lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9 #include <arpa/inet.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_prefetch.h>
  13 #include <rte_byteorder.h>
  14
  15 #include "rte_swx_pipeline.h"
  16 #include "rte_swx_ctl.h"
  17
  18 #define CHECK(condition, err_code)                                             \
  19 do {                                                                           \
  20         if (!(condition))                                                      \
  21                 return -(err_code);                                            \
  22 } while (0)
  23
  24 #define CHECK_NAME(name, err_code)                                             \
  25         CHECK((name) && (name)[0], err_code)
  26
  27 #ifndef TRACE_LEVEL
  28 #define TRACE_LEVEL 0
  29 #endif
  30
  31 #if TRACE_LEVEL
  32 #define TRACE(...) printf(__VA_ARGS__)
  33 #else
  34 #define TRACE(...)
  35 #endif
  36
  37 #define ntoh64(x) rte_be_to_cpu_64(x)
  38 #define hton64(x) rte_cpu_to_be_64(x)
  39
  40 /*
  41  * Struct.
  42  */
  43 struct field {
  44         char name[RTE_SWX_NAME_SIZE];
  45         uint32_t n_bits;
  46         uint32_t offset;
  47 };
  48
  49 struct struct_type {
  50         TAILQ_ENTRY(struct_type) node;
  51         char name[RTE_SWX_NAME_SIZE];
  52         struct field *fields;
  53         uint32_t n_fields;
  54         uint32_t n_bits;
  55 };
  56
  57 TAILQ_HEAD(struct_type_tailq, struct_type);
  58
  59 /*
  60  * Input port.
  61  */
  62 struct port_in_type {
  63         TAILQ_ENTRY(port_in_type) node;
  64         char name[RTE_SWX_NAME_SIZE];
  65         struct rte_swx_port_in_ops ops;
  66 };
  67
  68 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  69
  70 struct port_in {
  71         TAILQ_ENTRY(port_in) node;
  72         struct port_in_type *type;
  73         void *obj;
  74         uint32_t id;
  75 };
  76
  77 TAILQ_HEAD(port_in_tailq, port_in);
  78
  79 struct port_in_runtime {
  80         rte_swx_port_in_pkt_rx_t pkt_rx;
  81         void *obj;
  82 };
  83
  84 /*
  85  * Output port.
  86  */
  87 struct port_out_type {
  88         TAILQ_ENTRY(port_out_type) node;
  89         char name[RTE_SWX_NAME_SIZE];
  90         struct rte_swx_port_out_ops ops;
  91 };
  92
  93 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  94
  95 struct port_out {
  96         TAILQ_ENTRY(port_out) node;
  97         struct port_out_type *type;
  98         void *obj;
  99         uint32_t id;
 100 };
 101
 102 TAILQ_HEAD(port_out_tailq, port_out);
 103
 104 struct port_out_runtime {
 105         rte_swx_port_out_pkt_tx_t pkt_tx;
 106         rte_swx_port_out_flush_t flush;
 107         void *obj;
 108 };
 109
 110 /*
 111  * Extern object.
 112  */
 113 struct extern_type_member_func {
 114         TAILQ_ENTRY(extern_type_member_func) node;
 115         char name[RTE_SWX_NAME_SIZE];
 116         rte_swx_extern_type_member_func_t func;
 117         uint32_t id;
 118 };
 119
 120 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 121
 122 struct extern_type {
 123         TAILQ_ENTRY(extern_type) node;
 124         char name[RTE_SWX_NAME_SIZE];
 125         struct struct_type *mailbox_struct_type;
 126         rte_swx_extern_type_constructor_t constructor;
 127         rte_swx_extern_type_destructor_t destructor;
 128         struct extern_type_member_func_tailq funcs;
 129         uint32_t n_funcs;
 130 };
 131
 132 TAILQ_HEAD(extern_type_tailq, extern_type);
 133
 134 struct extern_obj {
 135         TAILQ_ENTRY(extern_obj) node;
 136         char name[RTE_SWX_NAME_SIZE];
 137         struct extern_type *type;
 138         void *obj;
 139         uint32_t struct_id;
 140         uint32_t id;
 141 };
 142
 143 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 144
 145 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 146 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 147 #endif
 148
 149 struct extern_obj_runtime {
 150         void *obj;
 151         uint8_t *mailbox;
 152         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 153 };
 154
 155 /*
 156  * Extern function.
 157  */
 158 struct extern_func {
 159         TAILQ_ENTRY(extern_func) node;
 160         char name[RTE_SWX_NAME_SIZE];
 161         struct struct_type *mailbox_struct_type;
 162         rte_swx_extern_func_t func;
 163         uint32_t struct_id;
 164         uint32_t id;
 165 };
 166
 167 TAILQ_HEAD(extern_func_tailq, extern_func);
 168
 169 struct extern_func_runtime {
 170         uint8_t *mailbox;
 171         rte_swx_extern_func_t func;
 172 };
 173
 174 /*
 175  * Header.
 176  */
 177 struct header {
 178         TAILQ_ENTRY(header) node;
 179         char name[RTE_SWX_NAME_SIZE];
 180         struct struct_type *st;
 181         uint32_t struct_id;
 182         uint32_t id;
 183 };
 184
 185 TAILQ_HEAD(header_tailq, header);
 186
 187 struct header_runtime {
 188         uint8_t *ptr0;
 189 };
 190
 191 struct header_out_runtime {
 192         uint8_t *ptr0;
 193         uint8_t *ptr;
 194         uint32_t n_bytes;
 195 };
 196
 197 /*
 198  * Instruction.
 199  */
 200
 201 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 202  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 203  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 204  * when transferred to packet meta-data and in NBO when transferred to packet
 205  * headers.
 206  */
 207
 208 /* Notation conventions:
 209  *    -Header field: H = h.header.field (dst/src)
 210  *    -Meta-data field: M = m.field (dst/src)
 211  *    -Extern object mailbox field: E = e.field (dst/src)
 212  *    -Extern function mailbox field: F = f.field (dst/src)
 213  *    -Table action data field: T = t.field (src only)
 214  *    -Immediate value: I = 32-bit unsigned value (src only)
 215  */
 216
 217 enum instruction_type {
 218         /* rx m.port_in */
 219         INSTR_RX,
 220
 221         /* tx m.port_out */
 222         INSTR_TX,
 223
 224         /* extract h.header */
 225         INSTR_HDR_EXTRACT,
 226         INSTR_HDR_EXTRACT2,
 227         INSTR_HDR_EXTRACT3,
 228         INSTR_HDR_EXTRACT4,
 229         INSTR_HDR_EXTRACT5,
 230         INSTR_HDR_EXTRACT6,
 231         INSTR_HDR_EXTRACT7,
 232         INSTR_HDR_EXTRACT8,
 233
 234         /* emit h.header */
 235         INSTR_HDR_EMIT,
 236         INSTR_HDR_EMIT_TX,
 237         INSTR_HDR_EMIT2_TX,
 238         INSTR_HDR_EMIT3_TX,
 239         INSTR_HDR_EMIT4_TX,
 240         INSTR_HDR_EMIT5_TX,
 241         INSTR_HDR_EMIT6_TX,
 242         INSTR_HDR_EMIT7_TX,
 243         INSTR_HDR_EMIT8_TX,
 244
 245         /* validate h.header */
 246         INSTR_HDR_VALIDATE,
 247
 248         /* invalidate h.header */
 249         INSTR_HDR_INVALIDATE,
 250
 251         /* mov dst src
 252          * dst = src
 253          * dst = HMEF, src = HMEFTI
 254          */
 255         INSTR_MOV,   /* dst = MEF, src = MEFT */
 256         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 257         INSTR_MOV_I, /* dst = HMEF, src = I */
 258
 259         /* dma h.header t.field
 260          * memcpy(h.header, t.field, sizeof(h.header))
 261          */
 262         INSTR_DMA_HT,
 263         INSTR_DMA_HT2,
 264         INSTR_DMA_HT3,
 265         INSTR_DMA_HT4,
 266         INSTR_DMA_HT5,
 267         INSTR_DMA_HT6,
 268         INSTR_DMA_HT7,
 269         INSTR_DMA_HT8,
 270
 271         /* add dst src
 272          * dst += src
 273          * dst = HMEF, src = HMEFTI
 274          */
 275         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 276         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 277         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 278         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 279         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 280         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 281
 282         /* sub dst src
 283          * dst -= src
 284          * dst = HMEF, src = HMEFTI
 285          */
 286         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 287         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 288         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 289         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 290         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 291         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 292
 293         /* ckadd dst src
 294          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 295          * dst = H, src = {H, h.header}
 296          */
 297         INSTR_ALU_CKADD_FIELD,    /* src = H */
 298         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 299         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 300
 301         /* cksub dst src
 302          * dst = dst '- src
 303          * dst = H, src = H
 304          */
 305         INSTR_ALU_CKSUB_FIELD,
 306
 307         /* and dst src
 308          * dst &= src
 309          * dst = HMEF, src = HMEFTI
 310          */
 311         INSTR_ALU_AND,   /* dst = MEF, src = MEFT */
 312         INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 313         INSTR_ALU_AND_I, /* dst = HMEF, src = I */
 314
 315         /* or dst src
 316          * dst |= src
 317          * dst = HMEF, src = HMEFTI
 318          */
 319         INSTR_ALU_OR,   /* dst = MEF, src = MEFT */
 320         INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 321         INSTR_ALU_OR_I, /* dst = HMEF, src = I */
 322
 323         /* xor dst src
 324          * dst ^= src
 325          * dst = HMEF, src = HMEFTI
 326          */
 327         INSTR_ALU_XOR,   /* dst = MEF, src = MEFT */
 328         INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 329         INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
 330
 331         /* shl dst src
 332          * dst <<= src
 333          * dst = HMEF, src = HMEFTI
 334          */
 335         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 336         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 337         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 338         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 339         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 340         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 341
 342         /* shr dst src
 343          * dst >>= src
 344          * dst = HMEF, src = HMEFTI
 345          */
 346         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 347         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 348         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 349         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 350         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 351         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 352
 353         /* table TABLE */
 354         INSTR_TABLE,
 355
 356         /* extern e.obj.func */
 357         INSTR_EXTERN_OBJ,
 358
 359         /* extern f.func */
 360         INSTR_EXTERN_FUNC,
 361
 362         /* jmp LABEL
 363          * Unconditional jump
 364          */
 365         INSTR_JMP,
 366
 367         /* jmpv LABEL h.header
 368          * Jump if header is valid
 369          */
 370         INSTR_JMP_VALID,
 371
 372         /* jmpnv LABEL h.header
 373          * Jump if header is invalid
 374          */
 375         INSTR_JMP_INVALID,
 376
 377         /* jmph LABEL
 378          * Jump if table lookup hit
 379          */
 380         INSTR_JMP_HIT,
 381
 382         /* jmpnh LABEL
 383          * Jump if table lookup miss
 384          */
 385         INSTR_JMP_MISS,
 386
 387         /* jmpa LABEL ACTION
 388          * Jump if action run
 389          */
 390         INSTR_JMP_ACTION_HIT,
 391
 392         /* jmpna LABEL ACTION
 393          * Jump if action not run
 394          */
 395         INSTR_JMP_ACTION_MISS,
 396
 397         /* jmpeq LABEL a b
 398          * Jump is a is equal to b
 399          * a = HMEFT, b = HMEFTI
 400          */
 401         INSTR_JMP_EQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
 402         INSTR_JMP_EQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
 403         INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 404
 405         /* jmpneq LABEL a b
 406          * Jump is a is not equal to b
 407          * a = HMEFT, b = HMEFTI
 408          */
 409         INSTR_JMP_NEQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
 410         INSTR_JMP_NEQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
 411         INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 412
 413         /* jmplt LABEL a b
 414          * Jump if a is less than b
 415          * a = HMEFT, b = HMEFTI
 416          */
 417         INSTR_JMP_LT,    /* a = MEF, b = MEF */
 418         INSTR_JMP_LT_MH, /* a = MEF, b = H */
 419         INSTR_JMP_LT_HM, /* a = H, b = MEF */
 420         INSTR_JMP_LT_HH, /* a = H, b = H */
 421         INSTR_JMP_LT_MI, /* a = MEF, b = I */
 422         INSTR_JMP_LT_HI, /* a = H, b = I */
 423
 424         /* jmpgt LABEL a b
 425          * Jump if a is greater than b
 426          * a = HMEFT, b = HMEFTI
 427          */
 428         INSTR_JMP_GT,    /* a = MEF, b = MEF */
 429         INSTR_JMP_GT_MH, /* a = MEF, b = H */
 430         INSTR_JMP_GT_HM, /* a = H, b = MEF */
 431         INSTR_JMP_GT_HH, /* a = H, b = H */
 432         INSTR_JMP_GT_MI, /* a = MEF, b = I */
 433         INSTR_JMP_GT_HI, /* a = H, b = I */
 434
 435         /* return
 436          * Return from action
 437          */
 438         INSTR_RETURN,
 439 };
 440
 441 struct instr_operand {
 442         uint8_t struct_id;
 443         uint8_t n_bits;
 444         uint8_t offset;
 445         uint8_t pad;
 446 };
 447
 448 struct instr_io {
 449         struct {
 450                 uint8_t offset;
 451                 uint8_t n_bits;
 452                 uint8_t pad[2];
 453         } io;
 454
 455         struct {
 456                 uint8_t header_id[8];
 457                 uint8_t struct_id[8];
 458                 uint8_t n_bytes[8];
 459         } hdr;
 460 };
 461
 462 struct instr_hdr_validity {
 463         uint8_t header_id;
 464 };
 465
 466 struct instr_table {
 467         uint8_t table_id;
 468 };
 469
 470 struct instr_extern_obj {
 471         uint8_t ext_obj_id;
 472         uint8_t func_id;
 473 };
 474
 475 struct instr_extern_func {
 476         uint8_t ext_func_id;
 477 };
 478
 479 struct instr_dst_src {
 480         struct instr_operand dst;
 481         union {
 482                 struct instr_operand src;
 483                 uint32_t src_val;
 484         };
 485 };
 486
 487 struct instr_dma {
 488         struct {
 489                 uint8_t header_id[8];
 490                 uint8_t struct_id[8];
 491         } dst;
 492
 493         struct {
 494                 uint8_t offset[8];
 495         } src;
 496
 497         uint16_t n_bytes[8];
 498 };
 499
 500 struct instr_jmp {
 501         struct instruction *ip;
 502
 503         union {
 504                 struct instr_operand a;
 505                 uint8_t header_id;
 506                 uint8_t action_id;
 507         };
 508
 509         union {
 510                 struct instr_operand b;
 511                 uint32_t b_val;
 512         };
 513 };
 514
 515 struct instruction {
 516         enum instruction_type type;
 517         union {
 518                 struct instr_io io;
 519                 struct instr_hdr_validity valid;
 520                 struct instr_dst_src mov;
 521                 struct instr_dma dma;
 522                 struct instr_dst_src alu;
 523                 struct instr_table table;
 524                 struct instr_extern_obj ext_obj;
 525                 struct instr_extern_func ext_func;
 526                 struct instr_jmp jmp;
 527         };
 528 };
 529
 530 struct instruction_data {
 531         char label[RTE_SWX_NAME_SIZE];
 532         char jmp_label[RTE_SWX_NAME_SIZE];
 533         uint32_t n_users; /* user = jmp instruction to this instruction. */
 534         int invalid;
 535 };
 536
 537 /*
 538  * Action.
 539  */
 540 struct action {
 541         TAILQ_ENTRY(action) node;
 542         char name[RTE_SWX_NAME_SIZE];
 543         struct struct_type *st;
 544         struct instruction *instructions;
 545         uint32_t n_instructions;
 546         uint32_t id;
 547 };
 548
 549 TAILQ_HEAD(action_tailq, action);
 550
 551 /*
 552  * Table.
 553  */
 554 struct table_type {
 555         TAILQ_ENTRY(table_type) node;
 556         char name[RTE_SWX_NAME_SIZE];
 557         enum rte_swx_table_match_type match_type;
 558         struct rte_swx_table_ops ops;
 559 };
 560
 561 TAILQ_HEAD(table_type_tailq, table_type);
 562
 563 struct match_field {
 564         enum rte_swx_table_match_type match_type;
 565         struct field *field;
 566 };
 567
 568 struct table {
 569         TAILQ_ENTRY(table) node;
 570         char name[RTE_SWX_NAME_SIZE];
 571         char args[RTE_SWX_NAME_SIZE];
 572         struct table_type *type; /* NULL when n_fields == 0. */
 573
 574         /* Match. */
 575         struct match_field *fields;
 576         uint32_t n_fields;
 577         int is_header; /* Only valid when n_fields > 0. */
 578         struct header *header; /* Only valid when n_fields > 0. */
 579
 580         /* Action. */
 581         struct action **actions;
 582         struct action *default_action;
 583         uint8_t *default_action_data;
 584         uint32_t n_actions;
 585         int default_action_is_const;
 586         uint32_t action_data_size_max;
 587
 588         uint32_t size;
 589         uint32_t id;
 590 };
 591
 592 TAILQ_HEAD(table_tailq, table);
 593
 594 struct table_runtime {
 595         rte_swx_table_lookup_t func;
 596         void *mailbox;
 597         uint8_t **key;
 598 };
 599
 600 /*
 601  * Pipeline.
 602  */
 603 struct thread {
 604         /* Packet. */
 605         struct rte_swx_pkt pkt;
 606         uint8_t *ptr;
 607
 608         /* Structures. */
 609         uint8_t **structs;
 610
 611         /* Packet headers. */
 612         struct header_runtime *headers; /* Extracted or generated headers. */
 613         struct header_out_runtime *headers_out; /* Emitted headers. */
 614         uint8_t *header_storage;
 615         uint8_t *header_out_storage;
 616         uint64_t valid_headers;
 617         uint32_t n_headers_out;
 618
 619         /* Packet meta-data. */
 620         uint8_t *metadata;
 621
 622         /* Tables. */
 623         struct table_runtime *tables;
 624         struct rte_swx_table_state *table_state;
 625         uint64_t action_id;
 626         int hit; /* 0 = Miss, 1 = Hit. */
 627
 628         /* Extern objects and functions. */
 629         struct extern_obj_runtime *extern_objs;
 630         struct extern_func_runtime *extern_funcs;
 631
 632         /* Instructions. */
 633         struct instruction *ip;
 634         struct instruction *ret;
 635 };
 636
 637 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 638 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 639 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 640
 641 #define HEADER_VALID(thread, header_id) \
 642         MASK64_BIT_GET((thread)->valid_headers, header_id)
 643
 644 #define ALU(thread, ip, operator)  \
 645 {                                                                              \
 646         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 647         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 648         uint64_t dst64 = *dst64_ptr;                                           \
 649         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 650         uint64_t dst = dst64 & dst64_mask;                                     \
 651                                                                                \
 652         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 653         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 654         uint64_t src64 = *src64_ptr;                                           \
 655         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 656         uint64_t src = src64 & src64_mask;                                     \
 657                                                                                \
 658         uint64_t result = dst operator src;                                    \
 659                                                                                \
 660         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 661 }
 662
 663 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 664
 665 #define ALU_S(thread, ip, operator)  \
 666 {                                                                              \
 667         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 668         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 669         uint64_t dst64 = *dst64_ptr;                                           \
 670         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 671         uint64_t dst = dst64 & dst64_mask;                                     \
 672                                                                                \
 673         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 674         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 675         uint64_t src64 = *src64_ptr;                                           \
 676         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 677                                                                                \
 678         uint64_t result = dst operator src;                                    \
 679                                                                                \
 680         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 681 }
 682
 683 #define ALU_MH ALU_S
 684
 685 #define ALU_HM(thread, ip, operator)  \
 686 {                                                                              \
 687         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 688         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 689         uint64_t dst64 = *dst64_ptr;                                           \
 690         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 691         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 692                                                                                \
 693         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 694         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 695         uint64_t src64 = *src64_ptr;                                           \
 696         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 697         uint64_t src = src64 & src64_mask;                                     \
 698                                                                                \
 699         uint64_t result = dst operator src;                                    \
 700         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 701                                                                                \
 702         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 703 }
 704
 705 #define ALU_HH(thread, ip, operator)  \
 706 {                                                                              \
 707         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 708         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 709         uint64_t dst64 = *dst64_ptr;                                           \
 710         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 711         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 712                                                                                \
 713         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 714         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 715         uint64_t src64 = *src64_ptr;                                           \
 716         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 717                                                                                \
 718         uint64_t result = dst operator src;                                    \
 719         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 720                                                                                \
 721         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 722 }
 723
 724 #else
 725
 726 #define ALU_S ALU
 727 #define ALU_MH ALU
 728 #define ALU_HM ALU
 729 #define ALU_HH ALU
 730
 731 #endif
 732
 733 #define ALU_I(thread, ip, operator)  \
 734 {                                                                              \
 735         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 736         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 737         uint64_t dst64 = *dst64_ptr;                                           \
 738         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 739         uint64_t dst = dst64 & dst64_mask;                                     \
 740                                                                                \
 741         uint64_t src = (ip)->alu.src_val;                                      \
 742                                                                                \
 743         uint64_t result = dst operator src;                                    \
 744                                                                                \
 745         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 746 }
 747
 748 #define ALU_MI ALU_I
 749
 750 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 751
 752 #define ALU_HI(thread, ip, operator)  \
 753 {                                                                              \
 754         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 755         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 756         uint64_t dst64 = *dst64_ptr;                                           \
 757         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 758         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 759                                                                                \
 760         uint64_t src = (ip)->alu.src_val;                                      \
 761                                                                                \
 762         uint64_t result = dst operator src;                                    \
 763         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 764                                                                                \
 765         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 766 }
 767
 768 #else
 769
 770 #define ALU_HI ALU_I
 771
 772 #endif
 773
 774 #define MOV(thread, ip)  \
 775 {                                                                              \
 776         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 777         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 778         uint64_t dst64 = *dst64_ptr;                                           \
 779         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 780                                                                                \
 781         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 782         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 783         uint64_t src64 = *src64_ptr;                                           \
 784         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
 785         uint64_t src = src64 & src64_mask;                                     \
 786                                                                                \
 787         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 788 }
 789
 790 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 791
 792 #define MOV_S(thread, ip)  \
 793 {                                                                              \
 794         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 795         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 796         uint64_t dst64 = *dst64_ptr;                                           \
 797         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 798                                                                                \
 799         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 800         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 801         uint64_t src64 = *src64_ptr;                                           \
 802         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
 803                                                                                \
 804         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 805 }
 806
 807 #else
 808
 809 #define MOV_S MOV
 810
 811 #endif
 812
 813 #define MOV_I(thread, ip)  \
 814 {                                                                              \
 815         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 816         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 817         uint64_t dst64 = *dst64_ptr;                                           \
 818         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 819                                                                                \
 820         uint64_t src = (ip)->mov.src_val;                                      \
 821                                                                                \
 822         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 823 }
 824
 825 #define JMP_CMP(thread, ip, operator)  \
 826 {                                                                              \
 827         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
 828         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
 829         uint64_t a64 = *a64_ptr;                                               \
 830         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
 831         uint64_t a = a64 & a64_mask;                                           \
 832                                                                                \
 833         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
 834         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
 835         uint64_t b64 = *b64_ptr;                                               \
 836         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
 837         uint64_t b = b64 & b64_mask;                                           \
 838                                                                                \
 839         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
 840 }
 841
 842 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 843
 844 #define JMP_CMP_S(thread, ip, operator)  \
 845 {                                                                              \
 846         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
 847         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
 848         uint64_t a64 = *a64_ptr;                                               \
 849         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
 850         uint64_t a = a64 & a64_mask;                                           \
 851                                                                                \
 852         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
 853         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
 854         uint64_t b64 = *b64_ptr;                                               \
 855         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
 856                                                                                \
 857         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
 858 }
 859
 860 #define JMP_CMP_MH JMP_CMP_S
 861
 862 #define JMP_CMP_HM(thread, ip, operator)  \
 863 {                                                                              \
 864         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
 865         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
 866         uint64_t a64 = *a64_ptr;                                               \
 867         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
 868                                                                                \
 869         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
 870         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
 871         uint64_t b64 = *b64_ptr;                                               \
 872         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
 873         uint64_t b = b64 & b64_mask;                                           \
 874                                                                                \
 875         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
 876 }
 877
 878 #define JMP_CMP_HH(thread, ip, operator)  \
 879 {                                                                              \
 880         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
 881         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
 882         uint64_t a64 = *a64_ptr;                                               \
 883         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
 884                                                                                \
 885         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
 886         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
 887         uint64_t b64 = *b64_ptr;                                               \
 888         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
 889                                                                                \
 890         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
 891 }
 892
 893 #else
 894
 895 #define JMP_CMP_S JMP_CMP
 896 #define JMP_CMP_MH JMP_CMP
 897 #define JMP_CMP_HM JMP_CMP
 898 #define JMP_CMP_HH JMP_CMP
 899
 900 #endif
 901
 902 #define JMP_CMP_I(thread, ip, operator)  \
 903 {                                                                              \
 904         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
 905         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
 906         uint64_t a64 = *a64_ptr;                                               \
 907         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
 908         uint64_t a = a64 & a64_mask;                                           \
 909                                                                                \
 910         uint64_t b = (ip)->jmp.b_val;                                          \
 911                                                                                \
 912         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
 913 }
 914
 915 #define JMP_CMP_MI JMP_CMP_I
 916
 917 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 918
 919 #define JMP_CMP_HI(thread, ip, operator)  \
 920 {                                                                              \
 921         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
 922         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
 923         uint64_t a64 = *a64_ptr;                                               \
 924         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
 925                                                                                \
 926         uint64_t b = (ip)->jmp.b_val;                                          \
 927                                                                                \
 928         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
 929 }
 930
 931 #else
 932
 933 #define JMP_CMP_HI JMP_CMP_I
 934
 935 #endif
 936
 937 #define METADATA_READ(thread, offset, n_bits)                                  \
 938 ({                                                                             \
 939         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 940         uint64_t m64 = *m64_ptr;                                               \
 941         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 942         (m64 & m64_mask);                                                      \
 943 })
 944
 945 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
 946 {                                                                              \
 947         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 948         uint64_t m64 = *m64_ptr;                                               \
 949         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 950                                                                                \
 951         uint64_t m_new = value;                                                \
 952                                                                                \
 953         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
 954 }
 955
 956 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 957 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 958 #endif
 959
 960 struct rte_swx_pipeline {
 961         struct struct_type_tailq struct_types;
 962         struct port_in_type_tailq port_in_types;
 963         struct port_in_tailq ports_in;
 964         struct port_out_type_tailq port_out_types;
 965         struct port_out_tailq ports_out;
 966         struct extern_type_tailq extern_types;
 967         struct extern_obj_tailq extern_objs;
 968         struct extern_func_tailq extern_funcs;
 969         struct header_tailq headers;
 970         struct struct_type *metadata_st;
 971         uint32_t metadata_struct_id;
 972         struct action_tailq actions;
 973         struct table_type_tailq table_types;
 974         struct table_tailq tables;
 975
 976         struct port_in_runtime *in;
 977         struct port_out_runtime *out;
 978         struct instruction **action_instructions;
 979         struct rte_swx_table_state *table_state;
 980         struct instruction *instructions;
 981         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 982
 983         uint32_t n_structs;
 984         uint32_t n_ports_in;
 985         uint32_t n_ports_out;
 986         uint32_t n_extern_objs;
 987         uint32_t n_extern_funcs;
 988         uint32_t n_actions;
 989         uint32_t n_tables;
 990         uint32_t n_headers;
 991         uint32_t thread_id;
 992         uint32_t port_id;
 993         uint32_t n_instructions;
 994         int build_done;
 995         int numa_node;
 996 };
 997
 998 /*
 999  * Struct.
1000  */
1001 static struct struct_type *
1002 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1003 {
1004         struct struct_type *elem;
1005
1006         TAILQ_FOREACH(elem, &p->struct_types, node)
1007                 if (strcmp(elem->name, name) == 0)
1008                         return elem;
1009
1010         return NULL;
1011 }
1012
1013 static struct field *
1014 struct_type_field_find(struct struct_type *st, const char *name)
1015 {
1016         uint32_t i;
1017
1018         for (i = 0; i < st->n_fields; i++) {
1019                 struct field *f = &st->fields[i];
1020
1021                 if (strcmp(f->name, name) == 0)
1022                         return f;
1023         }
1024
1025         return NULL;
1026 }
1027
1028 int
1029 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1030                                       const char *name,
1031                                       struct rte_swx_field_params *fields,
1032                                       uint32_t n_fields)
1033 {
1034         struct struct_type *st;
1035         uint32_t i;
1036
1037         CHECK(p, EINVAL);
1038         CHECK_NAME(name, EINVAL);
1039         CHECK(fields, EINVAL);
1040         CHECK(n_fields, EINVAL);
1041
1042         for (i = 0; i < n_fields; i++) {
1043                 struct rte_swx_field_params *f = &fields[i];
1044                 uint32_t j;
1045
1046                 CHECK_NAME(f->name, EINVAL);
1047                 CHECK(f->n_bits, EINVAL);
1048                 CHECK(f->n_bits <= 64, EINVAL);
1049                 CHECK((f->n_bits & 7) == 0, EINVAL);
1050
1051                 for (j = 0; j < i; j++) {
1052                         struct rte_swx_field_params *f_prev = &fields[j];
1053
1054                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
1055                 }
1056         }
1057
1058         CHECK(!struct_type_find(p, name), EEXIST);
1059
1060         /* Node allocation. */
1061         st = calloc(1, sizeof(struct struct_type));
1062         CHECK(st, ENOMEM);
1063
1064         st->fields = calloc(n_fields, sizeof(struct field));
1065         if (!st->fields) {
1066                 free(st);
1067                 CHECK(0, ENOMEM);
1068         }
1069
1070         /* Node initialization. */
1071         strcpy(st->name, name);
1072         for (i = 0; i < n_fields; i++) {
1073                 struct field *dst = &st->fields[i];
1074                 struct rte_swx_field_params *src = &fields[i];
1075
1076                 strcpy(dst->name, src->name);
1077                 dst->n_bits = src->n_bits;
1078                 dst->offset = st->n_bits;
1079
1080                 st->n_bits += src->n_bits;
1081         }
1082         st->n_fields = n_fields;
1083
1084         /* Node add to tailq. */
1085         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1086
1087         return 0;
1088 }
1089
1090 static int
1091 struct_build(struct rte_swx_pipeline *p)
1092 {
1093         uint32_t i;
1094
1095         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1096                 struct thread *t = &p->threads[i];
1097
1098                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1099                 CHECK(t->structs, ENOMEM);
1100         }
1101
1102         return 0;
1103 }
1104
1105 static void
1106 struct_build_free(struct rte_swx_pipeline *p)
1107 {
1108         uint32_t i;
1109
1110         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1111                 struct thread *t = &p->threads[i];
1112
1113                 free(t->structs);
1114                 t->structs = NULL;
1115         }
1116 }
1117
1118 static void
1119 struct_free(struct rte_swx_pipeline *p)
1120 {
1121         struct_build_free(p);
1122
1123         /* Struct types. */
1124         for ( ; ; ) {
1125                 struct struct_type *elem;
1126
1127                 elem = TAILQ_FIRST(&p->struct_types);
1128                 if (!elem)
1129                         break;
1130
1131                 TAILQ_REMOVE(&p->struct_types, elem, node);
1132                 free(elem->fields);
1133                 free(elem);
1134         }
1135 }
1136
1137 /*
1138  * Input port.
1139  */
1140 static struct port_in_type *
1141 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1142 {
1143         struct port_in_type *elem;
1144
1145         if (!name)
1146                 return NULL;
1147
1148         TAILQ_FOREACH(elem, &p->port_in_types, node)
1149                 if (strcmp(elem->name, name) == 0)
1150                         return elem;
1151
1152         return NULL;
1153 }
1154
1155 int
1156 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1157                                        const char *name,
1158                                        struct rte_swx_port_in_ops *ops)
1159 {
1160         struct port_in_type *elem;
1161
1162         CHECK(p, EINVAL);
1163         CHECK_NAME(name, EINVAL);
1164         CHECK(ops, EINVAL);
1165         CHECK(ops->create, EINVAL);
1166         CHECK(ops->free, EINVAL);
1167         CHECK(ops->pkt_rx, EINVAL);
1168         CHECK(ops->stats_read, EINVAL);
1169
1170         CHECK(!port_in_type_find(p, name), EEXIST);
1171
1172         /* Node allocation. */
1173         elem = calloc(1, sizeof(struct port_in_type));
1174         CHECK(elem, ENOMEM);
1175
1176         /* Node initialization. */
1177         strcpy(elem->name, name);
1178         memcpy(&elem->ops, ops, sizeof(*ops));
1179
1180         /* Node add to tailq. */
1181         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1182
1183         return 0;
1184 }
1185
1186 static struct port_in *
1187 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1188 {
1189         struct port_in *port;
1190
1191         TAILQ_FOREACH(port, &p->ports_in, node)
1192                 if (port->id == port_id)
1193                         return port;
1194
1195         return NULL;
1196 }
1197
1198 int
1199 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1200                                 uint32_t port_id,
1201                                 const char *port_type_name,
1202                                 void *args)
1203 {
1204         struct port_in_type *type = NULL;
1205         struct port_in *port = NULL;
1206         void *obj = NULL;
1207
1208         CHECK(p, EINVAL);
1209
1210         CHECK(!port_in_find(p, port_id), EINVAL);
1211
1212         CHECK_NAME(port_type_name, EINVAL);
1213         type = port_in_type_find(p, port_type_name);
1214         CHECK(type, EINVAL);
1215
1216         obj = type->ops.create(args);
1217         CHECK(obj, ENODEV);
1218
1219         /* Node allocation. */
1220         port = calloc(1, sizeof(struct port_in));
1221         CHECK(port, ENOMEM);
1222
1223         /* Node initialization. */
1224         port->type = type;
1225         port->obj = obj;
1226         port->id = port_id;
1227
1228         /* Node add to tailq. */
1229         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1230         if (p->n_ports_in < port_id + 1)
1231                 p->n_ports_in = port_id + 1;
1232
1233         return 0;
1234 }
1235
1236 static int
1237 port_in_build(struct rte_swx_pipeline *p)
1238 {
1239         struct port_in *port;
1240         uint32_t i;
1241
1242         CHECK(p->n_ports_in, EINVAL);
1243         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1244
1245         for (i = 0; i < p->n_ports_in; i++)
1246                 CHECK(port_in_find(p, i), EINVAL);
1247
1248         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1249         CHECK(p->in, ENOMEM);
1250
1251         TAILQ_FOREACH(port, &p->ports_in, node) {
1252                 struct port_in_runtime *in = &p->in[port->id];
1253
1254                 in->pkt_rx = port->type->ops.pkt_rx;
1255                 in->obj = port->obj;
1256         }
1257
1258         return 0;
1259 }
1260
1261 static void
1262 port_in_build_free(struct rte_swx_pipeline *p)
1263 {
1264         free(p->in);
1265         p->in = NULL;
1266 }
1267
1268 static void
1269 port_in_free(struct rte_swx_pipeline *p)
1270 {
1271         port_in_build_free(p);
1272
1273         /* Input ports. */
1274         for ( ; ; ) {
1275                 struct port_in *port;
1276
1277                 port = TAILQ_FIRST(&p->ports_in);
1278                 if (!port)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->ports_in, port, node);
1282                 port->type->ops.free(port->obj);
1283                 free(port);
1284         }
1285
1286         /* Input port types. */
1287         for ( ; ; ) {
1288                 struct port_in_type *elem;
1289
1290                 elem = TAILQ_FIRST(&p->port_in_types);
1291                 if (!elem)
1292                         break;
1293
1294                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1295                 free(elem);
1296         }
1297 }
1298
1299 /*
1300  * Output port.
1301  */
1302 static struct port_out_type *
1303 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1304 {
1305         struct port_out_type *elem;
1306
1307         if (!name)
1308                 return NULL;
1309
1310         TAILQ_FOREACH(elem, &p->port_out_types, node)
1311                 if (!strcmp(elem->name, name))
1312                         return elem;
1313
1314         return NULL;
1315 }
1316
1317 int
1318 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1319                                         const char *name,
1320                                         struct rte_swx_port_out_ops *ops)
1321 {
1322         struct port_out_type *elem;
1323
1324         CHECK(p, EINVAL);
1325         CHECK_NAME(name, EINVAL);
1326         CHECK(ops, EINVAL);
1327         CHECK(ops->create, EINVAL);
1328         CHECK(ops->free, EINVAL);
1329         CHECK(ops->pkt_tx, EINVAL);
1330         CHECK(ops->stats_read, EINVAL);
1331
1332         CHECK(!port_out_type_find(p, name), EEXIST);
1333
1334         /* Node allocation. */
1335         elem = calloc(1, sizeof(struct port_out_type));
1336         CHECK(elem, ENOMEM);
1337
1338         /* Node initialization. */
1339         strcpy(elem->name, name);
1340         memcpy(&elem->ops, ops, sizeof(*ops));
1341
1342         /* Node add to tailq. */
1343         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1344
1345         return 0;
1346 }
1347
1348 static struct port_out *
1349 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1350 {
1351         struct port_out *port;
1352
1353         TAILQ_FOREACH(port, &p->ports_out, node)
1354                 if (port->id == port_id)
1355                         return port;
1356
1357         return NULL;
1358 }
1359
1360 int
1361 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1362                                  uint32_t port_id,
1363                                  const char *port_type_name,
1364                                  void *args)
1365 {
1366         struct port_out_type *type = NULL;
1367         struct port_out *port = NULL;
1368         void *obj = NULL;
1369
1370         CHECK(p, EINVAL);
1371
1372         CHECK(!port_out_find(p, port_id), EINVAL);
1373
1374         CHECK_NAME(port_type_name, EINVAL);
1375         type = port_out_type_find(p, port_type_name);
1376         CHECK(type, EINVAL);
1377
1378         obj = type->ops.create(args);
1379         CHECK(obj, ENODEV);
1380
1381         /* Node allocation. */
1382         port = calloc(1, sizeof(struct port_out));
1383         CHECK(port, ENOMEM);
1384
1385         /* Node initialization. */
1386         port->type = type;
1387         port->obj = obj;
1388         port->id = port_id;
1389
1390         /* Node add to tailq. */
1391         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1392         if (p->n_ports_out < port_id + 1)
1393                 p->n_ports_out = port_id + 1;
1394
1395         return 0;
1396 }
1397
1398 static int
1399 port_out_build(struct rte_swx_pipeline *p)
1400 {
1401         struct port_out *port;
1402         uint32_t i;
1403
1404         CHECK(p->n_ports_out, EINVAL);
1405
1406         for (i = 0; i < p->n_ports_out; i++)
1407                 CHECK(port_out_find(p, i), EINVAL);
1408
1409         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1410         CHECK(p->out, ENOMEM);
1411
1412         TAILQ_FOREACH(port, &p->ports_out, node) {
1413                 struct port_out_runtime *out = &p->out[port->id];
1414
1415                 out->pkt_tx = port->type->ops.pkt_tx;
1416                 out->flush = port->type->ops.flush;
1417                 out->obj = port->obj;
1418         }
1419
1420         return 0;
1421 }
1422
1423 static void
1424 port_out_build_free(struct rte_swx_pipeline *p)
1425 {
1426         free(p->out);
1427         p->out = NULL;
1428 }
1429
1430 static void
1431 port_out_free(struct rte_swx_pipeline *p)
1432 {
1433         port_out_build_free(p);
1434
1435         /* Output ports. */
1436         for ( ; ; ) {
1437                 struct port_out *port;
1438
1439                 port = TAILQ_FIRST(&p->ports_out);
1440                 if (!port)
1441                         break;
1442
1443                 TAILQ_REMOVE(&p->ports_out, port, node);
1444                 port->type->ops.free(port->obj);
1445                 free(port);
1446         }
1447
1448         /* Output port types. */
1449         for ( ; ; ) {
1450                 struct port_out_type *elem;
1451
1452                 elem = TAILQ_FIRST(&p->port_out_types);
1453                 if (!elem)
1454                         break;
1455
1456                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1457                 free(elem);
1458         }
1459 }
1460
1461 /*
1462  * Extern object.
1463  */
1464 static struct extern_type *
1465 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1466 {
1467         struct extern_type *elem;
1468
1469         TAILQ_FOREACH(elem, &p->extern_types, node)
1470                 if (strcmp(elem->name, name) == 0)
1471                         return elem;
1472
1473         return NULL;
1474 }
1475
1476 static struct extern_type_member_func *
1477 extern_type_member_func_find(struct extern_type *type, const char *name)
1478 {
1479         struct extern_type_member_func *elem;
1480
1481         TAILQ_FOREACH(elem, &type->funcs, node)
1482                 if (strcmp(elem->name, name) == 0)
1483                         return elem;
1484
1485         return NULL;
1486 }
1487
1488 static struct extern_obj *
1489 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1490 {
1491         struct extern_obj *elem;
1492
1493         TAILQ_FOREACH(elem, &p->extern_objs, node)
1494                 if (strcmp(elem->name, name) == 0)
1495                         return elem;
1496
1497         return NULL;
1498 }
1499
1500 static struct extern_type_member_func *
1501 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1502                              const char *name,
1503                              struct extern_obj **obj)
1504 {
1505         struct extern_obj *object;
1506         struct extern_type_member_func *func;
1507         char *object_name, *func_name;
1508
1509         if (name[0] != 'e' || name[1] != '.')
1510                 return NULL;
1511
1512         object_name = strdup(&name[2]);
1513         if (!object_name)
1514                 return NULL;
1515
1516         func_name = strchr(object_name, '.');
1517         if (!func_name) {
1518                 free(object_name);
1519                 return NULL;
1520         }
1521
1522         *func_name = 0;
1523         func_name++;
1524
1525         object = extern_obj_find(p, object_name);
1526         if (!object) {
1527                 free(object_name);
1528                 return NULL;
1529         }
1530
1531         func = extern_type_member_func_find(object->type, func_name);
1532         if (!func) {
1533                 free(object_name);
1534                 return NULL;
1535         }
1536
1537         if (obj)
1538                 *obj = object;
1539
1540         free(object_name);
1541         return func;
1542 }
1543
1544 static struct field *
1545 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1546                                const char *name,
1547                                struct extern_obj **object)
1548 {
1549         struct extern_obj *obj;
1550         struct field *f;
1551         char *obj_name, *field_name;
1552
1553         if ((name[0] != 'e') || (name[1] != '.'))
1554                 return NULL;
1555
1556         obj_name = strdup(&name[2]);
1557         if (!obj_name)
1558                 return NULL;
1559
1560         field_name = strchr(obj_name, '.');
1561         if (!field_name) {
1562                 free(obj_name);
1563                 return NULL;
1564         }
1565
1566         *field_name = 0;
1567         field_name++;
1568
1569         obj = extern_obj_find(p, obj_name);
1570         if (!obj) {
1571                 free(obj_name);
1572                 return NULL;
1573         }
1574
1575         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1576         if (!f) {
1577                 free(obj_name);
1578                 return NULL;
1579         }
1580
1581         if (object)
1582                 *object = obj;
1583
1584         free(obj_name);
1585         return f;
1586 }
1587
1588 int
1589 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1590         const char *name,
1591         const char *mailbox_struct_type_name,
1592         rte_swx_extern_type_constructor_t constructor,
1593         rte_swx_extern_type_destructor_t destructor)
1594 {
1595         struct extern_type *elem;
1596         struct struct_type *mailbox_struct_type;
1597
1598         CHECK(p, EINVAL);
1599
1600         CHECK_NAME(name, EINVAL);
1601         CHECK(!extern_type_find(p, name), EEXIST);
1602
1603         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1604         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1605         CHECK(mailbox_struct_type, EINVAL);
1606
1607         CHECK(constructor, EINVAL);
1608         CHECK(destructor, EINVAL);
1609
1610         /* Node allocation. */
1611         elem = calloc(1, sizeof(struct extern_type));
1612         CHECK(elem, ENOMEM);
1613
1614         /* Node initialization. */
1615         strcpy(elem->name, name);
1616         elem->mailbox_struct_type = mailbox_struct_type;
1617         elem->constructor = constructor;
1618         elem->destructor = destructor;
1619         TAILQ_INIT(&elem->funcs);
1620
1621         /* Node add to tailq. */
1622         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1623
1624         return 0;
1625 }
1626
1627 int
1628 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1629         const char *extern_type_name,
1630         const char *name,
1631         rte_swx_extern_type_member_func_t member_func)
1632 {
1633         struct extern_type *type;
1634         struct extern_type_member_func *type_member;
1635
1636         CHECK(p, EINVAL);
1637
1638         CHECK(extern_type_name, EINVAL);
1639         type = extern_type_find(p, extern_type_name);
1640         CHECK(type, EINVAL);
1641         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1642
1643         CHECK(name, EINVAL);
1644         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1645
1646         CHECK(member_func, EINVAL);
1647
1648         /* Node allocation. */
1649         type_member = calloc(1, sizeof(struct extern_type_member_func));
1650         CHECK(type_member, ENOMEM);
1651
1652         /* Node initialization. */
1653         strcpy(type_member->name, name);
1654         type_member->func = member_func;
1655         type_member->id = type->n_funcs;
1656
1657         /* Node add to tailq. */
1658         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1659         type->n_funcs++;
1660
1661         return 0;
1662 }
1663
1664 int
1665 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1666                                       const char *extern_type_name,
1667                                       const char *name,
1668                                       const char *args)
1669 {
1670         struct extern_type *type;
1671         struct extern_obj *obj;
1672         void *obj_handle;
1673
1674         CHECK(p, EINVAL);
1675
1676         CHECK_NAME(extern_type_name, EINVAL);
1677         type = extern_type_find(p, extern_type_name);
1678         CHECK(type, EINVAL);
1679
1680         CHECK_NAME(name, EINVAL);
1681         CHECK(!extern_obj_find(p, name), EEXIST);
1682
1683         /* Node allocation. */
1684         obj = calloc(1, sizeof(struct extern_obj));
1685         CHECK(obj, ENOMEM);
1686
1687         /* Object construction. */
1688         obj_handle = type->constructor(args);
1689         if (!obj_handle) {
1690                 free(obj);
1691                 CHECK(0, ENODEV);
1692         }
1693
1694         /* Node initialization. */
1695         strcpy(obj->name, name);
1696         obj->type = type;
1697         obj->obj = obj_handle;
1698         obj->struct_id = p->n_structs;
1699         obj->id = p->n_extern_objs;
1700
1701         /* Node add to tailq. */
1702         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1703         p->n_extern_objs++;
1704         p->n_structs++;
1705
1706         return 0;
1707 }
1708
1709 static int
1710 extern_obj_build(struct rte_swx_pipeline *p)
1711 {
1712         uint32_t i;
1713
1714         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1715                 struct thread *t = &p->threads[i];
1716                 struct extern_obj *obj;
1717
1718                 t->extern_objs = calloc(p->n_extern_objs,
1719                                         sizeof(struct extern_obj_runtime));
1720                 CHECK(t->extern_objs, ENOMEM);
1721
1722                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1723                         struct extern_obj_runtime *r =
1724                                 &t->extern_objs[obj->id];
1725                         struct extern_type_member_func *func;
1726                         uint32_t mailbox_size =
1727                                 obj->type->mailbox_struct_type->n_bits / 8;
1728
1729                         r->obj = obj->obj;
1730
1731                         r->mailbox = calloc(1, mailbox_size);
1732                         CHECK(r->mailbox, ENOMEM);
1733
1734                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1735                                 r->funcs[func->id] = func->func;
1736
1737                         t->structs[obj->struct_id] = r->mailbox;
1738                 }
1739         }
1740
1741         return 0;
1742 }
1743
1744 static void
1745 extern_obj_build_free(struct rte_swx_pipeline *p)
1746 {
1747         uint32_t i;
1748
1749         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1750                 struct thread *t = &p->threads[i];
1751                 uint32_t j;
1752
1753                 if (!t->extern_objs)
1754                         continue;
1755
1756                 for (j = 0; j < p->n_extern_objs; j++) {
1757                         struct extern_obj_runtime *r = &t->extern_objs[j];
1758
1759                         free(r->mailbox);
1760                 }
1761
1762                 free(t->extern_objs);
1763                 t->extern_objs = NULL;
1764         }
1765 }
1766
1767 static void
1768 extern_obj_free(struct rte_swx_pipeline *p)
1769 {
1770         extern_obj_build_free(p);
1771
1772         /* Extern objects. */
1773         for ( ; ; ) {
1774                 struct extern_obj *elem;
1775
1776                 elem = TAILQ_FIRST(&p->extern_objs);
1777                 if (!elem)
1778                         break;
1779
1780                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1781                 if (elem->obj)
1782                         elem->type->destructor(elem->obj);
1783                 free(elem);
1784         }
1785
1786         /* Extern types. */
1787         for ( ; ; ) {
1788                 struct extern_type *elem;
1789
1790                 elem = TAILQ_FIRST(&p->extern_types);
1791                 if (!elem)
1792                         break;
1793
1794                 TAILQ_REMOVE(&p->extern_types, elem, node);
1795
1796                 for ( ; ; ) {
1797                         struct extern_type_member_func *func;
1798
1799                         func = TAILQ_FIRST(&elem->funcs);
1800                         if (!func)
1801                                 break;
1802
1803                         TAILQ_REMOVE(&elem->funcs, func, node);
1804                         free(func);
1805                 }
1806
1807                 free(elem);
1808         }
1809 }
1810
1811 /*
1812  * Extern function.
1813  */
1814 static struct extern_func *
1815 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1816 {
1817         struct extern_func *elem;
1818
1819         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1820                 if (strcmp(elem->name, name) == 0)
1821                         return elem;
1822
1823         return NULL;
1824 }
1825
1826 static struct extern_func *
1827 extern_func_parse(struct rte_swx_pipeline *p,
1828                   const char *name)
1829 {
1830         if (name[0] != 'f' || name[1] != '.')
1831                 return NULL;
1832
1833         return extern_func_find(p, &name[2]);
1834 }
1835
1836 static struct field *
1837 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1838                                 const char *name,
1839                                 struct extern_func **function)
1840 {
1841         struct extern_func *func;
1842         struct field *f;
1843         char *func_name, *field_name;
1844
1845         if ((name[0] != 'f') || (name[1] != '.'))
1846                 return NULL;
1847
1848         func_name = strdup(&name[2]);
1849         if (!func_name)
1850                 return NULL;
1851
1852         field_name = strchr(func_name, '.');
1853         if (!field_name) {
1854                 free(func_name);
1855                 return NULL;
1856         }
1857
1858         *field_name = 0;
1859         field_name++;
1860
1861         func = extern_func_find(p, func_name);
1862         if (!func) {
1863                 free(func_name);
1864                 return NULL;
1865         }
1866
1867         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1868         if (!f) {
1869                 free(func_name);
1870                 return NULL;
1871         }
1872
1873         if (function)
1874                 *function = func;
1875
1876         free(func_name);
1877         return f;
1878 }
1879
1880 int
1881 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1882                                       const char *name,
1883                                       const char *mailbox_struct_type_name,
1884                                       rte_swx_extern_func_t func)
1885 {
1886         struct extern_func *f;
1887         struct struct_type *mailbox_struct_type;
1888
1889         CHECK(p, EINVAL);
1890
1891         CHECK_NAME(name, EINVAL);
1892         CHECK(!extern_func_find(p, name), EEXIST);
1893
1894         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1895         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1896         CHECK(mailbox_struct_type, EINVAL);
1897
1898         CHECK(func, EINVAL);
1899
1900         /* Node allocation. */
1901         f = calloc(1, sizeof(struct extern_func));
1902         CHECK(func, ENOMEM);
1903
1904         /* Node initialization. */
1905         strcpy(f->name, name);
1906         f->mailbox_struct_type = mailbox_struct_type;
1907         f->func = func;
1908         f->struct_id = p->n_structs;
1909         f->id = p->n_extern_funcs;
1910
1911         /* Node add to tailq. */
1912         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1913         p->n_extern_funcs++;
1914         p->n_structs++;
1915
1916         return 0;
1917 }
1918
1919 static int
1920 extern_func_build(struct rte_swx_pipeline *p)
1921 {
1922         uint32_t i;
1923
1924         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1925                 struct thread *t = &p->threads[i];
1926                 struct extern_func *func;
1927
1928                 /* Memory allocation. */
1929                 t->extern_funcs = calloc(p->n_extern_funcs,
1930                                          sizeof(struct extern_func_runtime));
1931                 CHECK(t->extern_funcs, ENOMEM);
1932
1933                 /* Extern function. */
1934                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1935                         struct extern_func_runtime *r =
1936                                 &t->extern_funcs[func->id];
1937                         uint32_t mailbox_size =
1938                                 func->mailbox_struct_type->n_bits / 8;
1939
1940                         r->func = func->func;
1941
1942                         r->mailbox = calloc(1, mailbox_size);
1943                         CHECK(r->mailbox, ENOMEM);
1944
1945                         t->structs[func->struct_id] = r->mailbox;
1946                 }
1947         }
1948
1949         return 0;
1950 }
1951
1952 static void
1953 extern_func_build_free(struct rte_swx_pipeline *p)
1954 {
1955         uint32_t i;
1956
1957         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1958                 struct thread *t = &p->threads[i];
1959                 uint32_t j;
1960
1961                 if (!t->extern_funcs)
1962                         continue;
1963
1964                 for (j = 0; j < p->n_extern_funcs; j++) {
1965                         struct extern_func_runtime *r = &t->extern_funcs[j];
1966
1967                         free(r->mailbox);
1968                 }
1969
1970                 free(t->extern_funcs);
1971                 t->extern_funcs = NULL;
1972         }
1973 }
1974
1975 static void
1976 extern_func_free(struct rte_swx_pipeline *p)
1977 {
1978         extern_func_build_free(p);
1979
1980         for ( ; ; ) {
1981                 struct extern_func *elem;
1982
1983                 elem = TAILQ_FIRST(&p->extern_funcs);
1984                 if (!elem)
1985                         break;
1986
1987                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1988                 free(elem);
1989         }
1990 }
1991
1992 /*
1993  * Header.
1994  */
1995 static struct header *
1996 header_find(struct rte_swx_pipeline *p, const char *name)
1997 {
1998         struct header *elem;
1999
2000         TAILQ_FOREACH(elem, &p->headers, node)
2001                 if (strcmp(elem->name, name) == 0)
2002                         return elem;
2003
2004         return NULL;
2005 }
2006
2007 static struct header *
2008 header_parse(struct rte_swx_pipeline *p,
2009              const char *name)
2010 {
2011         if (name[0] != 'h' || name[1] != '.')
2012                 return NULL;
2013
2014         return header_find(p, &name[2]);
2015 }
2016
2017 static struct field *
2018 header_field_parse(struct rte_swx_pipeline *p,
2019                    const char *name,
2020                    struct header **header)
2021 {
2022         struct header *h;
2023         struct field *f;
2024         char *header_name, *field_name;
2025
2026         if ((name[0] != 'h') || (name[1] != '.'))
2027                 return NULL;
2028
2029         header_name = strdup(&name[2]);
2030         if (!header_name)
2031                 return NULL;
2032
2033         field_name = strchr(header_name, '.');
2034         if (!field_name) {
2035                 free(header_name);
2036                 return NULL;
2037         }
2038
2039         *field_name = 0;
2040         field_name++;
2041
2042         h = header_find(p, header_name);
2043         if (!h) {
2044                 free(header_name);
2045                 return NULL;
2046         }
2047
2048         f = struct_type_field_find(h->st, field_name);
2049         if (!f) {
2050                 free(header_name);
2051                 return NULL;
2052         }
2053
2054         if (header)
2055                 *header = h;
2056
2057         free(header_name);
2058         return f;
2059 }
2060
2061 int
2062 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2063                                         const char *name,
2064                                         const char *struct_type_name)
2065 {
2066         struct struct_type *st;
2067         struct header *h;
2068         size_t n_headers_max;
2069
2070         CHECK(p, EINVAL);
2071         CHECK_NAME(name, EINVAL);
2072         CHECK_NAME(struct_type_name, EINVAL);
2073
2074         CHECK(!header_find(p, name), EEXIST);
2075
2076         st = struct_type_find(p, struct_type_name);
2077         CHECK(st, EINVAL);
2078
2079         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2080         CHECK(p->n_headers < n_headers_max, ENOSPC);
2081
2082         /* Node allocation. */
2083         h = calloc(1, sizeof(struct header));
2084         CHECK(h, ENOMEM);
2085
2086         /* Node initialization. */
2087         strcpy(h->name, name);
2088         h->st = st;
2089         h->struct_id = p->n_structs;
2090         h->id = p->n_headers;
2091
2092         /* Node add to tailq. */
2093         TAILQ_INSERT_TAIL(&p->headers, h, node);
2094         p->n_headers++;
2095         p->n_structs++;
2096
2097         return 0;
2098 }
2099
2100 static int
2101 header_build(struct rte_swx_pipeline *p)
2102 {
2103         struct header *h;
2104         uint32_t n_bytes = 0, i;
2105
2106         TAILQ_FOREACH(h, &p->headers, node) {
2107                 n_bytes += h->st->n_bits / 8;
2108         }
2109
2110         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2111                 struct thread *t = &p->threads[i];
2112                 uint32_t offset = 0;
2113
2114                 t->headers = calloc(p->n_headers,
2115                                     sizeof(struct header_runtime));
2116                 CHECK(t->headers, ENOMEM);
2117
2118                 t->headers_out = calloc(p->n_headers,
2119                                         sizeof(struct header_out_runtime));
2120                 CHECK(t->headers_out, ENOMEM);
2121
2122                 t->header_storage = calloc(1, n_bytes);
2123                 CHECK(t->header_storage, ENOMEM);
2124
2125                 t->header_out_storage = calloc(1, n_bytes);
2126                 CHECK(t->header_out_storage, ENOMEM);
2127
2128                 TAILQ_FOREACH(h, &p->headers, node) {
2129                         uint8_t *header_storage;
2130
2131                         header_storage = &t->header_storage[offset];
2132                         offset += h->st->n_bits / 8;
2133
2134                         t->headers[h->id].ptr0 = header_storage;
2135                         t->structs[h->struct_id] = header_storage;
2136                 }
2137         }
2138
2139         return 0;
2140 }
2141
2142 static void
2143 header_build_free(struct rte_swx_pipeline *p)
2144 {
2145         uint32_t i;
2146
2147         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2148                 struct thread *t = &p->threads[i];
2149
2150                 free(t->headers_out);
2151                 t->headers_out = NULL;
2152
2153                 free(t->headers);
2154                 t->headers = NULL;
2155
2156                 free(t->header_out_storage);
2157                 t->header_out_storage = NULL;
2158
2159                 free(t->header_storage);
2160                 t->header_storage = NULL;
2161         }
2162 }
2163
2164 static void
2165 header_free(struct rte_swx_pipeline *p)
2166 {
2167         header_build_free(p);
2168
2169         for ( ; ; ) {
2170                 struct header *elem;
2171
2172                 elem = TAILQ_FIRST(&p->headers);
2173                 if (!elem)
2174                         break;
2175
2176                 TAILQ_REMOVE(&p->headers, elem, node);
2177                 free(elem);
2178         }
2179 }
2180
2181 /*
2182  * Meta-data.
2183  */
2184 static struct field *
2185 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2186 {
2187         if (!p->metadata_st)
2188                 return NULL;
2189
2190         if (name[0] != 'm' || name[1] != '.')
2191                 return NULL;
2192
2193         return struct_type_field_find(p->metadata_st, &name[2]);
2194 }
2195
2196 int
2197 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2198                                           const char *struct_type_name)
2199 {
2200         struct struct_type *st = NULL;
2201
2202         CHECK(p, EINVAL);
2203
2204         CHECK_NAME(struct_type_name, EINVAL);
2205         st  = struct_type_find(p, struct_type_name);
2206         CHECK(st, EINVAL);
2207         CHECK(!p->metadata_st, EINVAL);
2208
2209         p->metadata_st = st;
2210         p->metadata_struct_id = p->n_structs;
2211
2212         p->n_structs++;
2213
2214         return 0;
2215 }
2216
2217 static int
2218 metadata_build(struct rte_swx_pipeline *p)
2219 {
2220         uint32_t n_bytes = p->metadata_st->n_bits / 8;
2221         uint32_t i;
2222
2223         /* Thread-level initialization. */
2224         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2225                 struct thread *t = &p->threads[i];
2226                 uint8_t *metadata;
2227
2228                 metadata = calloc(1, n_bytes);
2229                 CHECK(metadata, ENOMEM);
2230
2231                 t->metadata = metadata;
2232                 t->structs[p->metadata_struct_id] = metadata;
2233         }
2234
2235         return 0;
2236 }
2237
2238 static void
2239 metadata_build_free(struct rte_swx_pipeline *p)
2240 {
2241         uint32_t i;
2242
2243         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2244                 struct thread *t = &p->threads[i];
2245
2246                 free(t->metadata);
2247                 t->metadata = NULL;
2248         }
2249 }
2250
2251 static void
2252 metadata_free(struct rte_swx_pipeline *p)
2253 {
2254         metadata_build_free(p);
2255 }
2256
2257 /*
2258  * Instruction.
2259  */
2260 static int
2261 instruction_is_jmp(struct instruction *instr)
2262 {
2263         switch (instr->type) {
2264         case INSTR_JMP:
2265         case INSTR_JMP_VALID:
2266         case INSTR_JMP_INVALID:
2267         case INSTR_JMP_HIT:
2268         case INSTR_JMP_MISS:
2269         case INSTR_JMP_ACTION_HIT:
2270         case INSTR_JMP_ACTION_MISS:
2271         case INSTR_JMP_EQ:
2272         case INSTR_JMP_EQ_S:
2273         case INSTR_JMP_EQ_I:
2274         case INSTR_JMP_NEQ:
2275         case INSTR_JMP_NEQ_S:
2276         case INSTR_JMP_NEQ_I:
2277         case INSTR_JMP_LT:
2278         case INSTR_JMP_LT_MH:
2279         case INSTR_JMP_LT_HM:
2280         case INSTR_JMP_LT_HH:
2281         case INSTR_JMP_LT_MI:
2282         case INSTR_JMP_LT_HI:
2283         case INSTR_JMP_GT:
2284         case INSTR_JMP_GT_MH:
2285         case INSTR_JMP_GT_HM:
2286         case INSTR_JMP_GT_HH:
2287         case INSTR_JMP_GT_MI:
2288         case INSTR_JMP_GT_HI:
2289                 return 1;
2290
2291         default:
2292                 return 0;
2293         }
2294 }
2295
2296 static struct field *
2297 action_field_parse(struct action *action, const char *name);
2298
2299 static struct field *
2300 struct_field_parse(struct rte_swx_pipeline *p,
2301                    struct action *action,
2302                    const char *name,
2303                    uint32_t *struct_id)
2304 {
2305         struct field *f;
2306
2307         switch (name[0]) {
2308         case 'h':
2309         {
2310                 struct header *header;
2311
2312                 f = header_field_parse(p, name, &header);
2313                 if (!f)
2314                         return NULL;
2315
2316                 *struct_id = header->struct_id;
2317                 return f;
2318         }
2319
2320         case 'm':
2321         {
2322                 f = metadata_field_parse(p, name);
2323                 if (!f)
2324                         return NULL;
2325
2326                 *struct_id = p->metadata_struct_id;
2327                 return f;
2328         }
2329
2330         case 't':
2331         {
2332                 if (!action)
2333                         return NULL;
2334
2335                 f = action_field_parse(action, name);
2336                 if (!f)
2337                         return NULL;
2338
2339                 *struct_id = 0;
2340                 return f;
2341         }
2342
2343         case 'e':
2344         {
2345                 struct extern_obj *obj;
2346
2347                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2348                 if (!f)
2349                         return NULL;
2350
2351                 *struct_id = obj->struct_id;
2352                 return f;
2353         }
2354
2355         case 'f':
2356         {
2357                 struct extern_func *func;
2358
2359                 f = extern_func_mailbox_field_parse(p, name, &func);
2360                 if (!f)
2361                         return NULL;
2362
2363                 *struct_id = func->struct_id;
2364                 return f;
2365         }
2366
2367         default:
2368                 return NULL;
2369         }
2370 }
2371
2372 static inline void
2373 pipeline_port_inc(struct rte_swx_pipeline *p)
2374 {
2375         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2376 }
2377
2378 static inline void
2379 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2380 {
2381         t->ip = p->instructions;
2382 }
2383
2384 static inline void
2385 thread_ip_set(struct thread *t, struct instruction *ip)
2386 {
2387         t->ip = ip;
2388 }
2389
2390 static inline void
2391 thread_ip_action_call(struct rte_swx_pipeline *p,
2392                       struct thread *t,
2393                       uint32_t action_id)
2394 {
2395         t->ret = t->ip + 1;
2396         t->ip = p->action_instructions[action_id];
2397 }
2398
2399 static inline void
2400 thread_ip_inc(struct rte_swx_pipeline *p);
2401
2402 static inline void
2403 thread_ip_inc(struct rte_swx_pipeline *p)
2404 {
2405         struct thread *t = &p->threads[p->thread_id];
2406
2407         t->ip++;
2408 }
2409
2410 static inline void
2411 thread_ip_inc_cond(struct thread *t, int cond)
2412 {
2413         t->ip += cond;
2414 }
2415
2416 static inline void
2417 thread_yield(struct rte_swx_pipeline *p)
2418 {
2419         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2420 }
2421
2422 static inline void
2423 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2424 {
2425         p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2426 }
2427
2428 /*
2429  * rx.
2430  */
2431 static int
2432 instr_rx_translate(struct rte_swx_pipeline *p,
2433                    struct action *action,
2434                    char **tokens,
2435                    int n_tokens,
2436                    struct instruction *instr,
2437                    struct instruction_data *data __rte_unused)
2438 {
2439         struct field *f;
2440
2441         CHECK(!action, EINVAL);
2442         CHECK(n_tokens == 2, EINVAL);
2443
2444         f = metadata_field_parse(p, tokens[1]);
2445         CHECK(f, EINVAL);
2446
2447         instr->type = INSTR_RX;
2448         instr->io.io.offset = f->offset / 8;
2449         instr->io.io.n_bits = f->n_bits;
2450         return 0;
2451 }
2452
2453 static inline void
2454 instr_rx_exec(struct rte_swx_pipeline *p);
2455
2456 static inline void
2457 instr_rx_exec(struct rte_swx_pipeline *p)
2458 {
2459         struct thread *t = &p->threads[p->thread_id];
2460         struct instruction *ip = t->ip;
2461         struct port_in_runtime *port = &p->in[p->port_id];
2462         struct rte_swx_pkt *pkt = &t->pkt;
2463         int pkt_received;
2464
2465         /* Packet. */
2466         pkt_received = port->pkt_rx(port->obj, pkt);
2467         t->ptr = &pkt->pkt[pkt->offset];
2468         rte_prefetch0(t->ptr);
2469
2470         TRACE("[Thread %2u] rx %s from port %u\n",
2471               p->thread_id,
2472               pkt_received ? "1 pkt" : "0 pkts",
2473               p->port_id);
2474
2475         /* Headers. */
2476         t->valid_headers = 0;
2477         t->n_headers_out = 0;
2478
2479         /* Meta-data. */
2480         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2481
2482         /* Tables. */
2483         t->table_state = p->table_state;
2484
2485         /* Thread. */
2486         pipeline_port_inc(p);
2487         thread_ip_inc_cond(t, pkt_received);
2488         thread_yield(p);
2489 }
2490
2491 /*
2492  * tx.
2493  */
2494 static int
2495 instr_tx_translate(struct rte_swx_pipeline *p,
2496                    struct action *action __rte_unused,
2497                    char **tokens,
2498                    int n_tokens,
2499                    struct instruction *instr,
2500                    struct instruction_data *data __rte_unused)
2501 {
2502         struct field *f;
2503
2504         CHECK(n_tokens == 2, EINVAL);
2505
2506         f = metadata_field_parse(p, tokens[1]);
2507         CHECK(f, EINVAL);
2508
2509         instr->type = INSTR_TX;
2510         instr->io.io.offset = f->offset / 8;
2511         instr->io.io.n_bits = f->n_bits;
2512         return 0;
2513 }
2514
2515 static inline void
2516 emit_handler(struct thread *t)
2517 {
2518         struct header_out_runtime *h0 = &t->headers_out[0];
2519         struct header_out_runtime *h1 = &t->headers_out[1];
2520         uint32_t offset = 0, i;
2521
2522         /* No header change or header decapsulation. */
2523         if ((t->n_headers_out == 1) &&
2524             (h0->ptr + h0->n_bytes == t->ptr)) {
2525                 TRACE("Emit handler: no header change or header decap.\n");
2526
2527                 t->pkt.offset -= h0->n_bytes;
2528                 t->pkt.length += h0->n_bytes;
2529
2530                 return;
2531         }
2532
2533         /* Header encapsulation (optionally, with prior header decasulation). */
2534         if ((t->n_headers_out == 2) &&
2535             (h1->ptr + h1->n_bytes == t->ptr) &&
2536             (h0->ptr == h0->ptr0)) {
2537                 uint32_t offset;
2538
2539                 TRACE("Emit handler: header encapsulation.\n");
2540
2541                 offset = h0->n_bytes + h1->n_bytes;
2542                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2543                 t->pkt.offset -= offset;
2544                 t->pkt.length += offset;
2545
2546                 return;
2547         }
2548
2549         /* Header insertion. */
2550         /* TBD */
2551
2552         /* Header extraction. */
2553         /* TBD */
2554
2555         /* For any other case. */
2556         TRACE("Emit handler: complex case.\n");
2557
2558         for (i = 0; i < t->n_headers_out; i++) {
2559                 struct header_out_runtime *h = &t->headers_out[i];
2560
2561                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2562                 offset += h->n_bytes;
2563         }
2564
2565         if (offset) {
2566                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2567                 t->pkt.offset -= offset;
2568                 t->pkt.length += offset;
2569         }
2570 }
2571
2572 static inline void
2573 instr_tx_exec(struct rte_swx_pipeline *p);
2574
2575 static inline void
2576 instr_tx_exec(struct rte_swx_pipeline *p)
2577 {
2578         struct thread *t = &p->threads[p->thread_id];
2579         struct instruction *ip = t->ip;
2580         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2581         struct port_out_runtime *port = &p->out[port_id];
2582         struct rte_swx_pkt *pkt = &t->pkt;
2583
2584         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2585               p->thread_id,
2586               (uint32_t)port_id);
2587
2588         /* Headers. */
2589         emit_handler(t);
2590
2591         /* Packet. */
2592         port->pkt_tx(port->obj, pkt);
2593
2594         /* Thread. */
2595         thread_ip_reset(p, t);
2596         instr_rx_exec(p);
2597 }
2598
2599 /*
2600  * extract.
2601  */
2602 static int
2603 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2604                             struct action *action,
2605                             char **tokens,
2606                             int n_tokens,
2607                             struct instruction *instr,
2608                             struct instruction_data *data __rte_unused)
2609 {
2610         struct header *h;
2611
2612         CHECK(!action, EINVAL);
2613         CHECK(n_tokens == 2, EINVAL);
2614
2615         h = header_parse(p, tokens[1]);
2616         CHECK(h, EINVAL);
2617
2618         instr->type = INSTR_HDR_EXTRACT;
2619         instr->io.hdr.header_id[0] = h->id;
2620         instr->io.hdr.struct_id[0] = h->struct_id;
2621         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2622         return 0;
2623 }
2624
2625 static inline void
2626 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2627
2628 static inline void
2629 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2630 {
2631         struct thread *t = &p->threads[p->thread_id];
2632         struct instruction *ip = t->ip;
2633         uint64_t valid_headers = t->valid_headers;
2634         uint8_t *ptr = t->ptr;
2635         uint32_t offset = t->pkt.offset;
2636         uint32_t length = t->pkt.length;
2637         uint32_t i;
2638
2639         for (i = 0; i < n_extract; i++) {
2640                 uint32_t header_id = ip->io.hdr.header_id[i];
2641                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2642                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2643
2644                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2645                       p->thread_id,
2646                       header_id,
2647                       n_bytes);
2648
2649                 /* Headers. */
2650                 t->structs[struct_id] = ptr;
2651                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2652
2653                 /* Packet. */
2654                 offset += n_bytes;
2655                 length -= n_bytes;
2656                 ptr += n_bytes;
2657         }
2658
2659         /* Headers. */
2660         t->valid_headers = valid_headers;
2661
2662         /* Packet. */
2663         t->pkt.offset = offset;
2664         t->pkt.length = length;
2665         t->ptr = ptr;
2666 }
2667
2668 static inline void
2669 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2670 {
2671         __instr_hdr_extract_exec(p, 1);
2672
2673         /* Thread. */
2674         thread_ip_inc(p);
2675 }
2676
2677 static inline void
2678 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2679 {
2680         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2681               p->thread_id);
2682
2683         __instr_hdr_extract_exec(p, 2);
2684
2685         /* Thread. */
2686         thread_ip_inc(p);
2687 }
2688
2689 static inline void
2690 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2691 {
2692         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2693               p->thread_id);
2694
2695         __instr_hdr_extract_exec(p, 3);
2696
2697         /* Thread. */
2698         thread_ip_inc(p);
2699 }
2700
2701 static inline void
2702 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2703 {
2704         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2705               p->thread_id);
2706
2707         __instr_hdr_extract_exec(p, 4);
2708
2709         /* Thread. */
2710         thread_ip_inc(p);
2711 }
2712
2713 static inline void
2714 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2715 {
2716         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2717               p->thread_id);
2718
2719         __instr_hdr_extract_exec(p, 5);
2720
2721         /* Thread. */
2722         thread_ip_inc(p);
2723 }
2724
2725 static inline void
2726 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2727 {
2728         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2729               p->thread_id);
2730
2731         __instr_hdr_extract_exec(p, 6);
2732
2733         /* Thread. */
2734         thread_ip_inc(p);
2735 }
2736
2737 static inline void
2738 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2739 {
2740         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2741               p->thread_id);
2742
2743         __instr_hdr_extract_exec(p, 7);
2744
2745         /* Thread. */
2746         thread_ip_inc(p);
2747 }
2748
2749 static inline void
2750 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2751 {
2752         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2753               p->thread_id);
2754
2755         __instr_hdr_extract_exec(p, 8);
2756
2757         /* Thread. */
2758         thread_ip_inc(p);
2759 }
2760
2761 /*
2762  * emit.
2763  */
2764 static int
2765 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2766                          struct action *action __rte_unused,
2767                          char **tokens,
2768                          int n_tokens,
2769                          struct instruction *instr,
2770                          struct instruction_data *data __rte_unused)
2771 {
2772         struct header *h;
2773
2774         CHECK(n_tokens == 2, EINVAL);
2775
2776         h = header_parse(p, tokens[1]);
2777         CHECK(h, EINVAL);
2778
2779         instr->type = INSTR_HDR_EMIT;
2780         instr->io.hdr.header_id[0] = h->id;
2781         instr->io.hdr.struct_id[0] = h->struct_id;
2782         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2783         return 0;
2784 }
2785
2786 static inline void
2787 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
2788
2789 static inline void
2790 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
2791 {
2792         struct thread *t = &p->threads[p->thread_id];
2793         struct instruction *ip = t->ip;
2794         uint32_t n_headers_out = t->n_headers_out;
2795         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
2796         uint8_t *ho_ptr = NULL;
2797         uint32_t ho_nbytes = 0, i;
2798
2799         for (i = 0; i < n_emit; i++) {
2800                 uint32_t header_id = ip->io.hdr.header_id[i];
2801                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2802                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2803
2804                 struct header_runtime *hi = &t->headers[header_id];
2805                 uint8_t *hi_ptr = t->structs[struct_id];
2806
2807                 TRACE("[Thread %2u]: emit header %u\n",
2808                       p->thread_id,
2809                       header_id);
2810
2811                 /* Headers. */
2812                 if (!i) {
2813                         if (!t->n_headers_out) {
2814                                 ho = &t->headers_out[0];
2815
2816                                 ho->ptr0 = hi->ptr0;
2817                                 ho->ptr = hi_ptr;
2818
2819                                 ho_ptr = hi_ptr;
2820                                 ho_nbytes = n_bytes;
2821
2822                                 n_headers_out = 1;
2823
2824                                 continue;
2825                         } else {
2826                                 ho_ptr = ho->ptr;
2827                                 ho_nbytes = ho->n_bytes;
2828                         }
2829                 }
2830
2831                 if (ho_ptr + ho_nbytes == hi_ptr) {
2832                         ho_nbytes += n_bytes;
2833                 } else {
2834                         ho->n_bytes = ho_nbytes;
2835
2836                         ho++;
2837                         ho->ptr0 = hi->ptr0;
2838                         ho->ptr = hi_ptr;
2839
2840                         ho_ptr = hi_ptr;
2841                         ho_nbytes = n_bytes;
2842
2843                         n_headers_out++;
2844                 }
2845         }
2846
2847         ho->n_bytes = ho_nbytes;
2848         t->n_headers_out = n_headers_out;
2849 }
2850
2851 static inline void
2852 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2853 {
2854         __instr_hdr_emit_exec(p, 1);
2855
2856         /* Thread. */
2857         thread_ip_inc(p);
2858 }
2859
2860 static inline void
2861 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2862 {
2863         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2864               p->thread_id);
2865
2866         __instr_hdr_emit_exec(p, 1);
2867         instr_tx_exec(p);
2868 }
2869
2870 static inline void
2871 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2872 {
2873         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2874               p->thread_id);
2875
2876         __instr_hdr_emit_exec(p, 2);
2877         instr_tx_exec(p);
2878 }
2879
2880 static inline void
2881 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2882 {
2883         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2884               p->thread_id);
2885
2886         __instr_hdr_emit_exec(p, 3);
2887         instr_tx_exec(p);
2888 }
2889
2890 static inline void
2891 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2892 {
2893         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2894               p->thread_id);
2895
2896         __instr_hdr_emit_exec(p, 4);
2897         instr_tx_exec(p);
2898 }
2899
2900 static inline void
2901 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2902 {
2903         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2904               p->thread_id);
2905
2906         __instr_hdr_emit_exec(p, 5);
2907         instr_tx_exec(p);
2908 }
2909
2910 static inline void
2911 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2912 {
2913         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2914               p->thread_id);
2915
2916         __instr_hdr_emit_exec(p, 6);
2917         instr_tx_exec(p);
2918 }
2919
2920 static inline void
2921 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2922 {
2923         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2924               p->thread_id);
2925
2926         __instr_hdr_emit_exec(p, 7);
2927         instr_tx_exec(p);
2928 }
2929
2930 static inline void
2931 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2932 {
2933         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2934               p->thread_id);
2935
2936         __instr_hdr_emit_exec(p, 8);
2937         instr_tx_exec(p);
2938 }
2939
2940 /*
2941  * validate.
2942  */
2943 static int
2944 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2945                              struct action *action __rte_unused,
2946                              char **tokens,
2947                              int n_tokens,
2948                              struct instruction *instr,
2949                              struct instruction_data *data __rte_unused)
2950 {
2951         struct header *h;
2952
2953         CHECK(n_tokens == 2, EINVAL);
2954
2955         h = header_parse(p, tokens[1]);
2956         CHECK(h, EINVAL);
2957
2958         instr->type = INSTR_HDR_VALIDATE;
2959         instr->valid.header_id = h->id;
2960         return 0;
2961 }
2962
2963 static inline void
2964 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2965 {
2966         struct thread *t = &p->threads[p->thread_id];
2967         struct instruction *ip = t->ip;
2968         uint32_t header_id = ip->valid.header_id;
2969
2970         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2971
2972         /* Headers. */
2973         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2974
2975         /* Thread. */
2976         thread_ip_inc(p);
2977 }
2978
2979 /*
2980  * invalidate.
2981  */
2982 static int
2983 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2984                                struct action *action __rte_unused,
2985                                char **tokens,
2986                                int n_tokens,
2987                                struct instruction *instr,
2988                                struct instruction_data *data __rte_unused)
2989 {
2990         struct header *h;
2991
2992         CHECK(n_tokens == 2, EINVAL);
2993
2994         h = header_parse(p, tokens[1]);
2995         CHECK(h, EINVAL);
2996
2997         instr->type = INSTR_HDR_INVALIDATE;
2998         instr->valid.header_id = h->id;
2999         return 0;
3000 }
3001
3002 static inline void
3003 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3004 {
3005         struct thread *t = &p->threads[p->thread_id];
3006         struct instruction *ip = t->ip;
3007         uint32_t header_id = ip->valid.header_id;
3008
3009         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3010
3011         /* Headers. */
3012         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3013
3014         /* Thread. */
3015         thread_ip_inc(p);
3016 }
3017
3018 /*
3019  * table.
3020  */
3021 static struct table *
3022 table_find(struct rte_swx_pipeline *p, const char *name);
3023
3024 static int
3025 instr_table_translate(struct rte_swx_pipeline *p,
3026                       struct action *action,
3027                       char **tokens,
3028                       int n_tokens,
3029                       struct instruction *instr,
3030                       struct instruction_data *data __rte_unused)
3031 {
3032         struct table *t;
3033
3034         CHECK(!action, EINVAL);
3035         CHECK(n_tokens == 2, EINVAL);
3036
3037         t = table_find(p, tokens[1]);
3038         CHECK(t, EINVAL);
3039
3040         instr->type = INSTR_TABLE;
3041         instr->table.table_id = t->id;
3042         return 0;
3043 }
3044
3045 static inline void
3046 instr_table_exec(struct rte_swx_pipeline *p)
3047 {
3048         struct thread *t = &p->threads[p->thread_id];
3049         struct instruction *ip = t->ip;
3050         uint32_t table_id = ip->table.table_id;
3051         struct rte_swx_table_state *ts = &t->table_state[table_id];
3052         struct table_runtime *table = &t->tables[table_id];
3053         uint64_t action_id;
3054         uint8_t *action_data;
3055         int done, hit;
3056
3057         /* Table. */
3058         done = table->func(ts->obj,
3059                            table->mailbox,
3060                            table->key,
3061                            &action_id,
3062                            &action_data,
3063                            &hit);
3064         if (!done) {
3065                 /* Thread. */
3066                 TRACE("[Thread %2u] table %u (not finalized)\n",
3067                       p->thread_id,
3068                       table_id);
3069
3070                 thread_yield(p);
3071                 return;
3072         }
3073
3074         action_id = hit ? action_id : ts->default_action_id;
3075         action_data = hit ? action_data : ts->default_action_data;
3076
3077         TRACE("[Thread %2u] table %u (%s, action %u)\n",
3078               p->thread_id,
3079               table_id,
3080               hit ? "hit" : "miss",
3081               (uint32_t)action_id);
3082
3083         t->action_id = action_id;
3084         t->structs[0] = action_data;
3085         t->hit = hit;
3086
3087         /* Thread. */
3088         thread_ip_action_call(p, t, action_id);
3089 }
3090
3091 /*
3092  * extern.
3093  */
3094 static int
3095 instr_extern_translate(struct rte_swx_pipeline *p,
3096                        struct action *action __rte_unused,
3097                        char **tokens,
3098                        int n_tokens,
3099                        struct instruction *instr,
3100                        struct instruction_data *data __rte_unused)
3101 {
3102         char *token = tokens[1];
3103
3104         CHECK(n_tokens == 2, EINVAL);
3105
3106         if (token[0] == 'e') {
3107                 struct extern_obj *obj;
3108                 struct extern_type_member_func *func;
3109
3110                 func = extern_obj_member_func_parse(p, token, &obj);
3111                 CHECK(func, EINVAL);
3112
3113                 instr->type = INSTR_EXTERN_OBJ;
3114                 instr->ext_obj.ext_obj_id = obj->id;
3115                 instr->ext_obj.func_id = func->id;
3116
3117                 return 0;
3118         }
3119
3120         if (token[0] == 'f') {
3121                 struct extern_func *func;
3122
3123                 func = extern_func_parse(p, token);
3124                 CHECK(func, EINVAL);
3125
3126                 instr->type = INSTR_EXTERN_FUNC;
3127                 instr->ext_func.ext_func_id = func->id;
3128
3129                 return 0;
3130         }
3131
3132         CHECK(0, EINVAL);
3133 }
3134
3135 static inline void
3136 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3137 {
3138         struct thread *t = &p->threads[p->thread_id];
3139         struct instruction *ip = t->ip;
3140         uint32_t obj_id = ip->ext_obj.ext_obj_id;
3141         uint32_t func_id = ip->ext_obj.func_id;
3142         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3143         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3144
3145         TRACE("[Thread %2u] extern obj %u member func %u\n",
3146               p->thread_id,
3147               obj_id,
3148               func_id);
3149
3150         /* Extern object member function execute. */
3151         uint32_t done = func(obj->obj, obj->mailbox);
3152
3153         /* Thread. */
3154         thread_ip_inc_cond(t, done);
3155         thread_yield_cond(p, done ^ 1);
3156 }
3157
3158 static inline void
3159 instr_extern_func_exec(struct rte_swx_pipeline *p)
3160 {
3161         struct thread *t = &p->threads[p->thread_id];
3162         struct instruction *ip = t->ip;
3163         uint32_t ext_func_id = ip->ext_func.ext_func_id;
3164         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3165         rte_swx_extern_func_t func = ext_func->func;
3166
3167         TRACE("[Thread %2u] extern func %u\n",
3168               p->thread_id,
3169               ext_func_id);
3170
3171         /* Extern function execute. */
3172         uint32_t done = func(ext_func->mailbox);
3173
3174         /* Thread. */
3175         thread_ip_inc_cond(t, done);
3176         thread_yield_cond(p, done ^ 1);
3177 }
3178
3179 /*
3180  * mov.
3181  */
3182 static int
3183 instr_mov_translate(struct rte_swx_pipeline *p,
3184                     struct action *action,
3185                     char **tokens,
3186                     int n_tokens,
3187                     struct instruction *instr,
3188                     struct instruction_data *data __rte_unused)
3189 {
3190         char *dst = tokens[1], *src = tokens[2];
3191         struct field *fdst, *fsrc;
3192         uint32_t dst_struct_id, src_struct_id, src_val;
3193
3194         CHECK(n_tokens == 3, EINVAL);
3195
3196         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3197         CHECK(fdst, EINVAL);
3198
3199         /* MOV or MOV_S. */
3200         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3201         if (fsrc) {
3202                 instr->type = INSTR_MOV;
3203                 if ((dst[0] == 'h' && src[0] != 'h') ||
3204                     (dst[0] != 'h' && src[0] == 'h'))
3205                         instr->type = INSTR_MOV_S;
3206
3207                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3208                 instr->mov.dst.n_bits = fdst->n_bits;
3209                 instr->mov.dst.offset = fdst->offset / 8;
3210                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3211                 instr->mov.src.n_bits = fsrc->n_bits;
3212                 instr->mov.src.offset = fsrc->offset / 8;
3213                 return 0;
3214         }
3215
3216         /* MOV_I. */
3217         src_val = strtoul(src, &src, 0);
3218         CHECK(!src[0], EINVAL);
3219
3220         if (dst[0] == 'h')
3221                 src_val = htonl(src_val);
3222
3223         instr->type = INSTR_MOV_I;
3224         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3225         instr->mov.dst.n_bits = fdst->n_bits;
3226         instr->mov.dst.offset = fdst->offset / 8;
3227         instr->mov.src_val = (uint32_t)src_val;
3228         return 0;
3229 }
3230
3231 static inline void
3232 instr_mov_exec(struct rte_swx_pipeline *p)
3233 {
3234         struct thread *t = &p->threads[p->thread_id];
3235         struct instruction *ip = t->ip;
3236
3237         TRACE("[Thread %2u] mov\n",
3238               p->thread_id);
3239
3240         MOV(t, ip);
3241
3242         /* Thread. */
3243         thread_ip_inc(p);
3244 }
3245
3246 static inline void
3247 instr_mov_s_exec(struct rte_swx_pipeline *p)
3248 {
3249         struct thread *t = &p->threads[p->thread_id];
3250         struct instruction *ip = t->ip;
3251
3252         TRACE("[Thread %2u] mov (s)\n",
3253               p->thread_id);
3254
3255         MOV_S(t, ip);
3256
3257         /* Thread. */
3258         thread_ip_inc(p);
3259 }
3260
3261 static inline void
3262 instr_mov_i_exec(struct rte_swx_pipeline *p)
3263 {
3264         struct thread *t = &p->threads[p->thread_id];
3265         struct instruction *ip = t->ip;
3266
3267         TRACE("[Thread %2u] mov m.f %x\n",
3268               p->thread_id,
3269               ip->mov.src_val);
3270
3271         MOV_I(t, ip);
3272
3273         /* Thread. */
3274         thread_ip_inc(p);
3275 }
3276
3277 /*
3278  * dma.
3279  */
3280 static int
3281 instr_dma_translate(struct rte_swx_pipeline *p,
3282                     struct action *action,
3283                     char **tokens,
3284                     int n_tokens,
3285                     struct instruction *instr,
3286                     struct instruction_data *data __rte_unused)
3287 {
3288         char *dst = tokens[1];
3289         char *src = tokens[2];
3290         struct header *h;
3291         struct field *tf;
3292
3293         CHECK(action, EINVAL);
3294         CHECK(n_tokens == 3, EINVAL);
3295
3296         h = header_parse(p, dst);
3297         CHECK(h, EINVAL);
3298
3299         tf = action_field_parse(action, src);
3300         CHECK(tf, EINVAL);
3301
3302         instr->type = INSTR_DMA_HT;
3303         instr->dma.dst.header_id[0] = h->id;
3304         instr->dma.dst.struct_id[0] = h->struct_id;
3305         instr->dma.n_bytes[0] = h->st->n_bits / 8;
3306         instr->dma.src.offset[0] = tf->offset / 8;
3307
3308         return 0;
3309 }
3310
3311 static inline void
3312 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3313
3314 static inline void
3315 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3316 {
3317         struct thread *t = &p->threads[p->thread_id];
3318         struct instruction *ip = t->ip;
3319         uint8_t *action_data = t->structs[0];
3320         uint64_t valid_headers = t->valid_headers;
3321         uint32_t i;
3322
3323         for (i = 0; i < n_dma; i++) {
3324                 uint32_t header_id = ip->dma.dst.header_id[i];
3325                 uint32_t struct_id = ip->dma.dst.struct_id[i];
3326                 uint32_t offset = ip->dma.src.offset[i];
3327                 uint32_t n_bytes = ip->dma.n_bytes[i];
3328
3329                 struct header_runtime *h = &t->headers[header_id];
3330                 uint8_t *h_ptr0 = h->ptr0;
3331                 uint8_t *h_ptr = t->structs[struct_id];
3332
3333                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3334                         h_ptr : h_ptr0;
3335                 void *src = &action_data[offset];
3336
3337                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3338
3339                 /* Headers. */
3340                 memcpy(dst, src, n_bytes);
3341                 t->structs[struct_id] = dst;
3342                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3343         }
3344
3345         t->valid_headers = valid_headers;
3346 }
3347
3348 static inline void
3349 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3350 {
3351         __instr_dma_ht_exec(p, 1);
3352
3353         /* Thread. */
3354         thread_ip_inc(p);
3355 }
3356
3357 static inline void
3358 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3359 {
3360         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3361               p->thread_id);
3362
3363         __instr_dma_ht_exec(p, 2);
3364
3365         /* Thread. */
3366         thread_ip_inc(p);
3367 }
3368
3369 static inline void
3370 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3371 {
3372         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3373               p->thread_id);
3374
3375         __instr_dma_ht_exec(p, 3);
3376
3377         /* Thread. */
3378         thread_ip_inc(p);
3379 }
3380
3381 static inline void
3382 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3383 {
3384         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3385               p->thread_id);
3386
3387         __instr_dma_ht_exec(p, 4);
3388
3389         /* Thread. */
3390         thread_ip_inc(p);
3391 }
3392
3393 static inline void
3394 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3395 {
3396         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3397               p->thread_id);
3398
3399         __instr_dma_ht_exec(p, 5);
3400
3401         /* Thread. */
3402         thread_ip_inc(p);
3403 }
3404
3405 static inline void
3406 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3407 {
3408         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3409               p->thread_id);
3410
3411         __instr_dma_ht_exec(p, 6);
3412
3413         /* Thread. */
3414         thread_ip_inc(p);
3415 }
3416
3417 static inline void
3418 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3419 {
3420         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3421               p->thread_id);
3422
3423         __instr_dma_ht_exec(p, 7);
3424
3425         /* Thread. */
3426         thread_ip_inc(p);
3427 }
3428
3429 static inline void
3430 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3431 {
3432         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3433               p->thread_id);
3434
3435         __instr_dma_ht_exec(p, 8);
3436
3437         /* Thread. */
3438         thread_ip_inc(p);
3439 }
3440
3441 /*
3442  * alu.
3443  */
3444 static int
3445 instr_alu_add_translate(struct rte_swx_pipeline *p,
3446                         struct action *action,
3447                         char **tokens,
3448                         int n_tokens,
3449                         struct instruction *instr,
3450                         struct instruction_data *data __rte_unused)
3451 {
3452         char *dst = tokens[1], *src = tokens[2];
3453         struct field *fdst, *fsrc;
3454         uint32_t dst_struct_id, src_struct_id, src_val;
3455
3456         CHECK(n_tokens == 3, EINVAL);
3457
3458         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3459         CHECK(fdst, EINVAL);
3460
3461         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3462         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3463         if (fsrc) {
3464                 instr->type = INSTR_ALU_ADD;
3465                 if (dst[0] == 'h' && src[0] == 'm')
3466                         instr->type = INSTR_ALU_ADD_HM;
3467                 if (dst[0] == 'm' && src[0] == 'h')
3468                         instr->type = INSTR_ALU_ADD_MH;
3469                 if (dst[0] == 'h' && src[0] == 'h')
3470                         instr->type = INSTR_ALU_ADD_HH;
3471
3472                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3473                 instr->alu.dst.n_bits = fdst->n_bits;
3474                 instr->alu.dst.offset = fdst->offset / 8;
3475                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3476                 instr->alu.src.n_bits = fsrc->n_bits;
3477                 instr->alu.src.offset = fsrc->offset / 8;
3478                 return 0;
3479         }
3480
3481         /* ADD_MI, ADD_HI. */
3482         src_val = strtoul(src, &src, 0);
3483         CHECK(!src[0], EINVAL);
3484
3485         instr->type = INSTR_ALU_ADD_MI;
3486         if (dst[0] == 'h')
3487                 instr->type = INSTR_ALU_ADD_HI;
3488
3489         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3490         instr->alu.dst.n_bits = fdst->n_bits;
3491         instr->alu.dst.offset = fdst->offset / 8;
3492         instr->alu.src_val = (uint32_t)src_val;
3493         return 0;
3494 }
3495
3496 static int
3497 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3498                         struct action *action,
3499                         char **tokens,
3500                         int n_tokens,
3501                         struct instruction *instr,
3502                         struct instruction_data *data __rte_unused)
3503 {
3504         char *dst = tokens[1], *src = tokens[2];
3505         struct field *fdst, *fsrc;
3506         uint32_t dst_struct_id, src_struct_id, src_val;
3507
3508         CHECK(n_tokens == 3, EINVAL);
3509
3510         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3511         CHECK(fdst, EINVAL);
3512
3513         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3514         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3515         if (fsrc) {
3516                 instr->type = INSTR_ALU_SUB;
3517                 if (dst[0] == 'h' && src[0] == 'm')
3518                         instr->type = INSTR_ALU_SUB_HM;
3519                 if (dst[0] == 'm' && src[0] == 'h')
3520                         instr->type = INSTR_ALU_SUB_MH;
3521                 if (dst[0] == 'h' && src[0] == 'h')
3522                         instr->type = INSTR_ALU_SUB_HH;
3523
3524                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3525                 instr->alu.dst.n_bits = fdst->n_bits;
3526                 instr->alu.dst.offset = fdst->offset / 8;
3527                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3528                 instr->alu.src.n_bits = fsrc->n_bits;
3529                 instr->alu.src.offset = fsrc->offset / 8;
3530                 return 0;
3531         }
3532
3533         /* SUB_MI, SUB_HI. */
3534         src_val = strtoul(src, &src, 0);
3535         CHECK(!src[0], EINVAL);
3536
3537         instr->type = INSTR_ALU_SUB_MI;
3538         if (dst[0] == 'h')
3539                 instr->type = INSTR_ALU_SUB_HI;
3540
3541         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3542         instr->alu.dst.n_bits = fdst->n_bits;
3543         instr->alu.dst.offset = fdst->offset / 8;
3544         instr->alu.src_val = (uint32_t)src_val;
3545         return 0;
3546 }
3547
3548 static int
3549 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3550                           struct action *action __rte_unused,
3551                           char **tokens,
3552                           int n_tokens,
3553                           struct instruction *instr,
3554                           struct instruction_data *data __rte_unused)
3555 {
3556         char *dst = tokens[1], *src = tokens[2];
3557         struct header *hdst, *hsrc;
3558         struct field *fdst, *fsrc;
3559
3560         CHECK(n_tokens == 3, EINVAL);
3561
3562         fdst = header_field_parse(p, dst, &hdst);
3563         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3564
3565         /* CKADD_FIELD. */
3566         fsrc = header_field_parse(p, src, &hsrc);
3567         if (fsrc) {
3568                 instr->type = INSTR_ALU_CKADD_FIELD;
3569                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3570                 instr->alu.dst.n_bits = fdst->n_bits;
3571                 instr->alu.dst.offset = fdst->offset / 8;
3572                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3573                 instr->alu.src.n_bits = fsrc->n_bits;
3574                 instr->alu.src.offset = fsrc->offset / 8;
3575                 return 0;
3576         }
3577
3578         /* CKADD_STRUCT, CKADD_STRUCT20. */
3579         hsrc = header_parse(p, src);
3580         CHECK(hsrc, EINVAL);
3581
3582         instr->type = INSTR_ALU_CKADD_STRUCT;
3583         if ((hsrc->st->n_bits / 8) == 20)
3584                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3585
3586         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3587         instr->alu.dst.n_bits = fdst->n_bits;
3588         instr->alu.dst.offset = fdst->offset / 8;
3589         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3590         instr->alu.src.n_bits = hsrc->st->n_bits;
3591         instr->alu.src.offset = 0; /* Unused. */
3592         return 0;
3593 }
3594
3595 static int
3596 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3597                           struct action *action __rte_unused,
3598                           char **tokens,
3599                           int n_tokens,
3600                           struct instruction *instr,
3601                           struct instruction_data *data __rte_unused)
3602 {
3603         char *dst = tokens[1], *src = tokens[2];
3604         struct header *hdst, *hsrc;
3605         struct field *fdst, *fsrc;
3606
3607         CHECK(n_tokens == 3, EINVAL);
3608
3609         fdst = header_field_parse(p, dst, &hdst);
3610         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3611
3612         fsrc = header_field_parse(p, src, &hsrc);
3613         CHECK(fsrc, EINVAL);
3614
3615         instr->type = INSTR_ALU_CKSUB_FIELD;
3616         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3617         instr->alu.dst.n_bits = fdst->n_bits;
3618         instr->alu.dst.offset = fdst->offset / 8;
3619         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3620         instr->alu.src.n_bits = fsrc->n_bits;
3621         instr->alu.src.offset = fsrc->offset / 8;
3622         return 0;
3623 }
3624
3625 static int
3626 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3627                         struct action *action,
3628                         char **tokens,
3629                         int n_tokens,
3630                         struct instruction *instr,
3631                         struct instruction_data *data __rte_unused)
3632 {
3633         char *dst = tokens[1], *src = tokens[2];
3634         struct field *fdst, *fsrc;
3635         uint32_t dst_struct_id, src_struct_id, src_val;
3636
3637         CHECK(n_tokens == 3, EINVAL);
3638
3639         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3640         CHECK(fdst, EINVAL);
3641
3642         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3643         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3644         if (fsrc) {
3645                 instr->type = INSTR_ALU_SHL;
3646                 if (dst[0] == 'h' && src[0] == 'm')
3647                         instr->type = INSTR_ALU_SHL_HM;
3648                 if (dst[0] == 'm' && src[0] == 'h')
3649                         instr->type = INSTR_ALU_SHL_MH;
3650                 if (dst[0] == 'h' && src[0] == 'h')
3651                         instr->type = INSTR_ALU_SHL_HH;
3652
3653                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3654                 instr->alu.dst.n_bits = fdst->n_bits;
3655                 instr->alu.dst.offset = fdst->offset / 8;
3656                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3657                 instr->alu.src.n_bits = fsrc->n_bits;
3658                 instr->alu.src.offset = fsrc->offset / 8;
3659                 return 0;
3660         }
3661
3662         /* SHL_MI, SHL_HI. */
3663         src_val = strtoul(src, &src, 0);
3664         CHECK(!src[0], EINVAL);
3665
3666         instr->type = INSTR_ALU_SHL_MI;
3667         if (dst[0] == 'h')
3668                 instr->type = INSTR_ALU_SHL_HI;
3669
3670         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3671         instr->alu.dst.n_bits = fdst->n_bits;
3672         instr->alu.dst.offset = fdst->offset / 8;
3673         instr->alu.src_val = (uint32_t)src_val;
3674         return 0;
3675 }
3676
3677 static int
3678 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3679                         struct action *action,
3680                         char **tokens,
3681                         int n_tokens,
3682                         struct instruction *instr,
3683                         struct instruction_data *data __rte_unused)
3684 {
3685         char *dst = tokens[1], *src = tokens[2];
3686         struct field *fdst, *fsrc;
3687         uint32_t dst_struct_id, src_struct_id, src_val;
3688
3689         CHECK(n_tokens == 3, EINVAL);
3690
3691         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3692         CHECK(fdst, EINVAL);
3693
3694         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3695         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3696         if (fsrc) {
3697                 instr->type = INSTR_ALU_SHR;
3698                 if (dst[0] == 'h' && src[0] == 'm')
3699                         instr->type = INSTR_ALU_SHR_HM;
3700                 if (dst[0] == 'm' && src[0] == 'h')
3701                         instr->type = INSTR_ALU_SHR_MH;
3702                 if (dst[0] == 'h' && src[0] == 'h')
3703                         instr->type = INSTR_ALU_SHR_HH;
3704
3705                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3706                 instr->alu.dst.n_bits = fdst->n_bits;
3707                 instr->alu.dst.offset = fdst->offset / 8;
3708                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3709                 instr->alu.src.n_bits = fsrc->n_bits;
3710                 instr->alu.src.offset = fsrc->offset / 8;
3711                 return 0;
3712         }
3713
3714         /* SHR_MI, SHR_HI. */
3715         src_val = strtoul(src, &src, 0);
3716         CHECK(!src[0], EINVAL);
3717
3718         instr->type = INSTR_ALU_SHR_MI;
3719         if (dst[0] == 'h')
3720                 instr->type = INSTR_ALU_SHR_HI;
3721
3722         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3723         instr->alu.dst.n_bits = fdst->n_bits;
3724         instr->alu.dst.offset = fdst->offset / 8;
3725         instr->alu.src_val = (uint32_t)src_val;
3726         return 0;
3727 }
3728
3729 static int
3730 instr_alu_and_translate(struct rte_swx_pipeline *p,
3731                         struct action *action,
3732                         char **tokens,
3733                         int n_tokens,
3734                         struct instruction *instr,
3735                         struct instruction_data *data __rte_unused)
3736 {
3737         char *dst = tokens[1], *src = tokens[2];
3738         struct field *fdst, *fsrc;
3739         uint32_t dst_struct_id, src_struct_id, src_val;
3740
3741         CHECK(n_tokens == 3, EINVAL);
3742
3743         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3744         CHECK(fdst, EINVAL);
3745
3746         /* AND or AND_S. */
3747         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3748         if (fsrc) {
3749                 instr->type = INSTR_ALU_AND;
3750                 if ((dst[0] == 'h' && src[0] != 'h') ||
3751                     (dst[0] != 'h' && src[0] == 'h'))
3752                         instr->type = INSTR_ALU_AND_S;
3753
3754                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3755                 instr->alu.dst.n_bits = fdst->n_bits;
3756                 instr->alu.dst.offset = fdst->offset / 8;
3757                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3758                 instr->alu.src.n_bits = fsrc->n_bits;
3759                 instr->alu.src.offset = fsrc->offset / 8;
3760                 return 0;
3761         }
3762
3763         /* AND_I. */
3764         src_val = strtoul(src, &src, 0);
3765         CHECK(!src[0], EINVAL);
3766
3767         if (dst[0] == 'h')
3768                 src_val = htonl(src_val);
3769
3770         instr->type = INSTR_ALU_AND_I;
3771         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3772         instr->alu.dst.n_bits = fdst->n_bits;
3773         instr->alu.dst.offset = fdst->offset / 8;
3774         instr->alu.src_val = (uint32_t)src_val;
3775         return 0;
3776 }
3777
3778 static int
3779 instr_alu_or_translate(struct rte_swx_pipeline *p,
3780                        struct action *action,
3781                        char **tokens,
3782                        int n_tokens,
3783                        struct instruction *instr,
3784                        struct instruction_data *data __rte_unused)
3785 {
3786         char *dst = tokens[1], *src = tokens[2];
3787         struct field *fdst, *fsrc;
3788         uint32_t dst_struct_id, src_struct_id, src_val;
3789
3790         CHECK(n_tokens == 3, EINVAL);
3791
3792         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3793         CHECK(fdst, EINVAL);
3794
3795         /* OR or OR_S. */
3796         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3797         if (fsrc) {
3798                 instr->type = INSTR_ALU_OR;
3799                 if ((dst[0] == 'h' && src[0] != 'h') ||
3800                     (dst[0] != 'h' && src[0] == 'h'))
3801                         instr->type = INSTR_ALU_OR_S;
3802
3803                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3804                 instr->alu.dst.n_bits = fdst->n_bits;
3805                 instr->alu.dst.offset = fdst->offset / 8;
3806                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3807                 instr->alu.src.n_bits = fsrc->n_bits;
3808                 instr->alu.src.offset = fsrc->offset / 8;
3809                 return 0;
3810         }
3811
3812         /* OR_I. */
3813         src_val = strtoul(src, &src, 0);
3814         CHECK(!src[0], EINVAL);
3815
3816         if (dst[0] == 'h')
3817                 src_val = htonl(src_val);
3818
3819         instr->type = INSTR_ALU_OR_I;
3820         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3821         instr->alu.dst.n_bits = fdst->n_bits;
3822         instr->alu.dst.offset = fdst->offset / 8;
3823         instr->alu.src_val = (uint32_t)src_val;
3824         return 0;
3825 }
3826
3827 static int
3828 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3829                         struct action *action,
3830                         char **tokens,
3831                         int n_tokens,
3832                         struct instruction *instr,
3833                         struct instruction_data *data __rte_unused)
3834 {
3835         char *dst = tokens[1], *src = tokens[2];
3836         struct field *fdst, *fsrc;
3837         uint32_t dst_struct_id, src_struct_id, src_val;
3838
3839         CHECK(n_tokens == 3, EINVAL);
3840
3841         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3842         CHECK(fdst, EINVAL);
3843
3844         /* XOR or XOR_S. */
3845         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3846         if (fsrc) {
3847                 instr->type = INSTR_ALU_XOR;
3848                 if ((dst[0] == 'h' && src[0] != 'h') ||
3849                     (dst[0] != 'h' && src[0] == 'h'))
3850                         instr->type = INSTR_ALU_XOR_S;
3851
3852                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3853                 instr->alu.dst.n_bits = fdst->n_bits;
3854                 instr->alu.dst.offset = fdst->offset / 8;
3855                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3856                 instr->alu.src.n_bits = fsrc->n_bits;
3857                 instr->alu.src.offset = fsrc->offset / 8;
3858                 return 0;
3859         }
3860
3861         /* XOR_I. */
3862         src_val = strtoul(src, &src, 0);
3863         CHECK(!src[0], EINVAL);
3864
3865         if (dst[0] == 'h')
3866                 src_val = htonl(src_val);
3867
3868         instr->type = INSTR_ALU_XOR_I;
3869         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3870         instr->alu.dst.n_bits = fdst->n_bits;
3871         instr->alu.dst.offset = fdst->offset / 8;
3872         instr->alu.src_val = (uint32_t)src_val;
3873         return 0;
3874 }
3875
3876 static inline void
3877 instr_alu_add_exec(struct rte_swx_pipeline *p)
3878 {
3879         struct thread *t = &p->threads[p->thread_id];
3880         struct instruction *ip = t->ip;
3881
3882         TRACE("[Thread %2u] add\n", p->thread_id);
3883
3884         /* Structs. */
3885         ALU(t, ip, +);
3886
3887         /* Thread. */
3888         thread_ip_inc(p);
3889 }
3890
3891 static inline void
3892 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3893 {
3894         struct thread *t = &p->threads[p->thread_id];
3895         struct instruction *ip = t->ip;
3896
3897         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3898
3899         /* Structs. */
3900         ALU_MH(t, ip, +);
3901
3902         /* Thread. */
3903         thread_ip_inc(p);
3904 }
3905
3906 static inline void
3907 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3908 {
3909         struct thread *t = &p->threads[p->thread_id];
3910         struct instruction *ip = t->ip;
3911
3912         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3913
3914         /* Structs. */
3915         ALU_HM(t, ip, +);
3916
3917         /* Thread. */
3918         thread_ip_inc(p);
3919 }
3920
3921 static inline void
3922 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3923 {
3924         struct thread *t = &p->threads[p->thread_id];
3925         struct instruction *ip = t->ip;
3926
3927         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3928
3929         /* Structs. */
3930         ALU_HH(t, ip, +);
3931
3932         /* Thread. */
3933         thread_ip_inc(p);
3934 }
3935
3936 static inline void
3937 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3938 {
3939         struct thread *t = &p->threads[p->thread_id];
3940         struct instruction *ip = t->ip;
3941
3942         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3943
3944         /* Structs. */
3945         ALU_MI(t, ip, +);
3946
3947         /* Thread. */
3948         thread_ip_inc(p);
3949 }
3950
3951 static inline void
3952 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3953 {
3954         struct thread *t = &p->threads[p->thread_id];
3955         struct instruction *ip = t->ip;
3956
3957         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3958
3959         /* Structs. */
3960         ALU_HI(t, ip, +);
3961
3962         /* Thread. */
3963         thread_ip_inc(p);
3964 }
3965
3966 static inline void
3967 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3968 {
3969         struct thread *t = &p->threads[p->thread_id];
3970         struct instruction *ip = t->ip;
3971
3972         TRACE("[Thread %2u] sub\n", p->thread_id);
3973
3974         /* Structs. */
3975         ALU(t, ip, -);
3976
3977         /* Thread. */
3978         thread_ip_inc(p);
3979 }
3980
3981 static inline void
3982 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3983 {
3984         struct thread *t = &p->threads[p->thread_id];
3985         struct instruction *ip = t->ip;
3986
3987         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3988
3989         /* Structs. */
3990         ALU_MH(t, ip, -);
3991
3992         /* Thread. */
3993         thread_ip_inc(p);
3994 }
3995
3996 static inline void
3997 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3998 {
3999         struct thread *t = &p->threads[p->thread_id];
4000         struct instruction *ip = t->ip;
4001
4002         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4003
4004         /* Structs. */
4005         ALU_HM(t, ip, -);
4006
4007         /* Thread. */
4008         thread_ip_inc(p);
4009 }
4010
4011 static inline void
4012 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4013 {
4014         struct thread *t = &p->threads[p->thread_id];
4015         struct instruction *ip = t->ip;
4016
4017         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4018
4019         /* Structs. */
4020         ALU_HH(t, ip, -);
4021
4022         /* Thread. */
4023         thread_ip_inc(p);
4024 }
4025
4026 static inline void
4027 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4028 {
4029         struct thread *t = &p->threads[p->thread_id];
4030         struct instruction *ip = t->ip;
4031
4032         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4033
4034         /* Structs. */
4035         ALU_MI(t, ip, -);
4036
4037         /* Thread. */
4038         thread_ip_inc(p);
4039 }
4040
4041 static inline void
4042 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4043 {
4044         struct thread *t = &p->threads[p->thread_id];
4045         struct instruction *ip = t->ip;
4046
4047         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4048
4049         /* Structs. */
4050         ALU_HI(t, ip, -);
4051
4052         /* Thread. */
4053         thread_ip_inc(p);
4054 }
4055
4056 static inline void
4057 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4058 {
4059         struct thread *t = &p->threads[p->thread_id];
4060         struct instruction *ip = t->ip;
4061
4062         TRACE("[Thread %2u] shl\n", p->thread_id);
4063
4064         /* Structs. */
4065         ALU(t, ip, <<);
4066
4067         /* Thread. */
4068         thread_ip_inc(p);
4069 }
4070
4071 static inline void
4072 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4073 {
4074         struct thread *t = &p->threads[p->thread_id];
4075         struct instruction *ip = t->ip;
4076
4077         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4078
4079         /* Structs. */
4080         ALU_MH(t, ip, <<);
4081
4082         /* Thread. */
4083         thread_ip_inc(p);
4084 }
4085
4086 static inline void
4087 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4088 {
4089         struct thread *t = &p->threads[p->thread_id];
4090         struct instruction *ip = t->ip;
4091
4092         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4093
4094         /* Structs. */
4095         ALU_HM(t, ip, <<);
4096
4097         /* Thread. */
4098         thread_ip_inc(p);
4099 }
4100
4101 static inline void
4102 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4103 {
4104         struct thread *t = &p->threads[p->thread_id];
4105         struct instruction *ip = t->ip;
4106
4107         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4108
4109         /* Structs. */
4110         ALU_HH(t, ip, <<);
4111
4112         /* Thread. */
4113         thread_ip_inc(p);
4114 }
4115
4116 static inline void
4117 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4118 {
4119         struct thread *t = &p->threads[p->thread_id];
4120         struct instruction *ip = t->ip;
4121
4122         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4123
4124         /* Structs. */
4125         ALU_MI(t, ip, <<);
4126
4127         /* Thread. */
4128         thread_ip_inc(p);
4129 }
4130
4131 static inline void
4132 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4133 {
4134         struct thread *t = &p->threads[p->thread_id];
4135         struct instruction *ip = t->ip;
4136
4137         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4138
4139         /* Structs. */
4140         ALU_HI(t, ip, <<);
4141
4142         /* Thread. */
4143         thread_ip_inc(p);
4144 }
4145
4146 static inline void
4147 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4148 {
4149         struct thread *t = &p->threads[p->thread_id];
4150         struct instruction *ip = t->ip;
4151
4152         TRACE("[Thread %2u] shr\n", p->thread_id);
4153
4154         /* Structs. */
4155         ALU(t, ip, >>);
4156
4157         /* Thread. */
4158         thread_ip_inc(p);
4159 }
4160
4161 static inline void
4162 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4163 {
4164         struct thread *t = &p->threads[p->thread_id];
4165         struct instruction *ip = t->ip;
4166
4167         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4168
4169         /* Structs. */
4170         ALU_MH(t, ip, >>);
4171
4172         /* Thread. */
4173         thread_ip_inc(p);
4174 }
4175
4176 static inline void
4177 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4178 {
4179         struct thread *t = &p->threads[p->thread_id];
4180         struct instruction *ip = t->ip;
4181
4182         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4183
4184         /* Structs. */
4185         ALU_HM(t, ip, >>);
4186
4187         /* Thread. */
4188         thread_ip_inc(p);
4189 }
4190
4191 static inline void
4192 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4193 {
4194         struct thread *t = &p->threads[p->thread_id];
4195         struct instruction *ip = t->ip;
4196
4197         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4198
4199         /* Structs. */
4200         ALU_HH(t, ip, >>);
4201
4202         /* Thread. */
4203         thread_ip_inc(p);
4204 }
4205
4206 static inline void
4207 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4208 {
4209         struct thread *t = &p->threads[p->thread_id];
4210         struct instruction *ip = t->ip;
4211
4212         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4213
4214         /* Structs. */
4215         ALU_MI(t, ip, >>);
4216
4217         /* Thread. */
4218         thread_ip_inc(p);
4219 }
4220
4221 static inline void
4222 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4223 {
4224         struct thread *t = &p->threads[p->thread_id];
4225         struct instruction *ip = t->ip;
4226
4227         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4228
4229         /* Structs. */
4230         ALU_HI(t, ip, >>);
4231
4232         /* Thread. */
4233         thread_ip_inc(p);
4234 }
4235
4236 static inline void
4237 instr_alu_and_exec(struct rte_swx_pipeline *p)
4238 {
4239         struct thread *t = &p->threads[p->thread_id];
4240         struct instruction *ip = t->ip;
4241
4242         TRACE("[Thread %2u] and\n", p->thread_id);
4243
4244         /* Structs. */
4245         ALU(t, ip, &);
4246
4247         /* Thread. */
4248         thread_ip_inc(p);
4249 }
4250
4251 static inline void
4252 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
4253 {
4254         struct thread *t = &p->threads[p->thread_id];
4255         struct instruction *ip = t->ip;
4256
4257         TRACE("[Thread %2u] and (s)\n", p->thread_id);
4258
4259         /* Structs. */
4260         ALU_S(t, ip, &);
4261
4262         /* Thread. */
4263         thread_ip_inc(p);
4264 }
4265
4266 static inline void
4267 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4268 {
4269         struct thread *t = &p->threads[p->thread_id];
4270         struct instruction *ip = t->ip;
4271
4272         TRACE("[Thread %2u] and (i)\n", p->thread_id);
4273
4274         /* Structs. */
4275         ALU_I(t, ip, &);
4276
4277         /* Thread. */
4278         thread_ip_inc(p);
4279 }
4280
4281 static inline void
4282 instr_alu_or_exec(struct rte_swx_pipeline *p)
4283 {
4284         struct thread *t = &p->threads[p->thread_id];
4285         struct instruction *ip = t->ip;
4286
4287         TRACE("[Thread %2u] or\n", p->thread_id);
4288
4289         /* Structs. */
4290         ALU(t, ip, |);
4291
4292         /* Thread. */
4293         thread_ip_inc(p);
4294 }
4295
4296 static inline void
4297 instr_alu_or_s_exec(struct rte_swx_pipeline *p)
4298 {
4299         struct thread *t = &p->threads[p->thread_id];
4300         struct instruction *ip = t->ip;
4301
4302         TRACE("[Thread %2u] or (s)\n", p->thread_id);
4303
4304         /* Structs. */
4305         ALU_S(t, ip, |);
4306
4307         /* Thread. */
4308         thread_ip_inc(p);
4309 }
4310
4311 static inline void
4312 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4313 {
4314         struct thread *t = &p->threads[p->thread_id];
4315         struct instruction *ip = t->ip;
4316
4317         TRACE("[Thread %2u] or (i)\n", p->thread_id);
4318
4319         /* Structs. */
4320         ALU_I(t, ip, |);
4321
4322         /* Thread. */
4323         thread_ip_inc(p);
4324 }
4325
4326 static inline void
4327 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4328 {
4329         struct thread *t = &p->threads[p->thread_id];
4330         struct instruction *ip = t->ip;
4331
4332         TRACE("[Thread %2u] xor\n", p->thread_id);
4333
4334         /* Structs. */
4335         ALU(t, ip, ^);
4336
4337         /* Thread. */
4338         thread_ip_inc(p);
4339 }
4340
4341 static inline void
4342 instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
4343 {
4344         struct thread *t = &p->threads[p->thread_id];
4345         struct instruction *ip = t->ip;
4346
4347         TRACE("[Thread %2u] xor (s)\n", p->thread_id);
4348
4349         /* Structs. */
4350         ALU_S(t, ip, ^);
4351
4352         /* Thread. */
4353         thread_ip_inc(p);
4354 }
4355
4356 static inline void
4357 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4358 {
4359         struct thread *t = &p->threads[p->thread_id];
4360         struct instruction *ip = t->ip;
4361
4362         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
4363
4364         /* Structs. */
4365         ALU_I(t, ip, ^);
4366
4367         /* Thread. */
4368         thread_ip_inc(p);
4369 }
4370
4371 static inline void
4372 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4373 {
4374         struct thread *t = &p->threads[p->thread_id];
4375         struct instruction *ip = t->ip;
4376         uint8_t *dst_struct, *src_struct;
4377         uint16_t *dst16_ptr, dst;
4378         uint64_t *src64_ptr, src64, src64_mask, src;
4379         uint64_t r;
4380
4381         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
4382
4383         /* Structs. */
4384         dst_struct = t->structs[ip->alu.dst.struct_id];
4385         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4386         dst = *dst16_ptr;
4387
4388         src_struct = t->structs[ip->alu.src.struct_id];
4389         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4390         src64 = *src64_ptr;
4391         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4392         src = src64 & src64_mask;
4393
4394         r = dst;
4395         r = ~r & 0xFFFF;
4396
4397         /* The first input (r) is a 16-bit number. The second and the third
4398          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
4399          * three numbers (output r) is a 34-bit number.
4400          */
4401         r += (src >> 32) + (src & 0xFFFFFFFF);
4402
4403         /* The first input is a 16-bit number. The second input is an 18-bit
4404          * number. In the worst case scenario, the sum of the two numbers is a
4405          * 19-bit number.
4406          */
4407         r = (r & 0xFFFF) + (r >> 16);
4408
4409         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4410          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
4411          */
4412         r = (r & 0xFFFF) + (r >> 16);
4413
4414         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4415          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4416          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
4417          * therefore the output r is always a 16-bit number.
4418          */
4419         r = (r & 0xFFFF) + (r >> 16);
4420
4421         r = ~r & 0xFFFF;
4422         r = r ? r : 0xFFFF;
4423
4424         *dst16_ptr = (uint16_t)r;
4425
4426         /* Thread. */
4427         thread_ip_inc(p);
4428 }
4429
4430 static inline void
4431 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4432 {
4433         struct thread *t = &p->threads[p->thread_id];
4434         struct instruction *ip = t->ip;
4435         uint8_t *dst_struct, *src_struct;
4436         uint16_t *dst16_ptr, dst;
4437         uint64_t *src64_ptr, src64, src64_mask, src;
4438         uint64_t r;
4439
4440         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
4441
4442         /* Structs. */
4443         dst_struct = t->structs[ip->alu.dst.struct_id];
4444         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4445         dst = *dst16_ptr;
4446
4447         src_struct = t->structs[ip->alu.src.struct_id];
4448         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4449         src64 = *src64_ptr;
4450         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4451         src = src64 & src64_mask;
4452
4453         r = dst;
4454         r = ~r & 0xFFFF;
4455
4456         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
4457          * the following sequence of operations in 2's complement arithmetic:
4458          *    a '- b = (a - b) % 0xFFFF.
4459          *
4460          * In order to prevent an underflow for the below subtraction, in which
4461          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
4462          * minuend), we first add a multiple of the 0xFFFF modulus to the
4463          * minuend. The number we add to the minuend needs to be a 34-bit number
4464          * or higher, so for readability reasons we picked the 36-bit multiple.
4465          * We are effectively turning the 16-bit minuend into a 36-bit number:
4466          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
4467          */
4468         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
4469
4470         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
4471          * result (the output r) is a 36-bit number.
4472          */
4473         r -= (src >> 32) + (src & 0xFFFFFFFF);
4474
4475         /* The first input is a 16-bit number. The second input is a 20-bit
4476          * number. Their sum is a 21-bit number.
4477          */
4478         r = (r & 0xFFFF) + (r >> 16);
4479
4480         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4481          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
4482          */
4483         r = (r & 0xFFFF) + (r >> 16);
4484
4485         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4486          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4487          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4488          * generated, therefore the output r is always a 16-bit number.
4489          */
4490         r = (r & 0xFFFF) + (r >> 16);
4491
4492         r = ~r & 0xFFFF;
4493         r = r ? r : 0xFFFF;
4494
4495         *dst16_ptr = (uint16_t)r;
4496
4497         /* Thread. */
4498         thread_ip_inc(p);
4499 }
4500
4501 static inline void
4502 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4503 {
4504         struct thread *t = &p->threads[p->thread_id];
4505         struct instruction *ip = t->ip;
4506         uint8_t *dst_struct, *src_struct;
4507         uint16_t *dst16_ptr;
4508         uint32_t *src32_ptr;
4509         uint64_t r0, r1;
4510
4511         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
4512
4513         /* Structs. */
4514         dst_struct = t->structs[ip->alu.dst.struct_id];
4515         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4516
4517         src_struct = t->structs[ip->alu.src.struct_id];
4518         src32_ptr = (uint32_t *)&src_struct[0];
4519
4520         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
4521         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
4522         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
4523         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
4524         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
4525
4526         /* The first input is a 16-bit number. The second input is a 19-bit
4527          * number. Their sum is a 20-bit number.
4528          */
4529         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4530
4531         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4532          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
4533          */
4534         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4535
4536         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4537          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4538          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
4539          * generated, therefore the output r is always a 16-bit number.
4540          */
4541         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4542
4543         r0 = ~r0 & 0xFFFF;
4544         r0 = r0 ? r0 : 0xFFFF;
4545
4546         *dst16_ptr = (uint16_t)r0;
4547
4548         /* Thread. */
4549         thread_ip_inc(p);
4550 }
4551
4552 static inline void
4553 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4554 {
4555         struct thread *t = &p->threads[p->thread_id];
4556         struct instruction *ip = t->ip;
4557         uint8_t *dst_struct, *src_struct;
4558         uint16_t *dst16_ptr;
4559         uint32_t *src32_ptr;
4560         uint64_t r = 0;
4561         uint32_t i;
4562
4563         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
4564
4565         /* Structs. */
4566         dst_struct = t->structs[ip->alu.dst.struct_id];
4567         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4568
4569         src_struct = t->structs[ip->alu.src.struct_id];
4570         src32_ptr = (uint32_t *)&src_struct[0];
4571
4572         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
4573          * Therefore, in the worst case scenario, a 35-bit number is added to a
4574          * 16-bit number (the input r), so the output r is 36-bit number.
4575          */
4576         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
4577                 r += *src32_ptr;
4578
4579         /* The first input is a 16-bit number. The second input is a 20-bit
4580          * number. Their sum is a 21-bit number.
4581          */
4582         r = (r & 0xFFFF) + (r >> 16);
4583
4584         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4585          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
4586          */
4587         r = (r & 0xFFFF) + (r >> 16);
4588
4589         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4590          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4591          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4592          * generated, therefore the output r is always a 16-bit number.
4593          */
4594         r = (r & 0xFFFF) + (r >> 16);
4595
4596         r = ~r & 0xFFFF;
4597         r = r ? r : 0xFFFF;
4598
4599         *dst16_ptr = (uint16_t)r;
4600
4601         /* Thread. */
4602         thread_ip_inc(p);
4603 }
4604
4605 /*
4606  * jmp.
4607  */
4608 static struct action *
4609 action_find(struct rte_swx_pipeline *p, const char *name);
4610
4611 static int
4612 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4613                     struct action *action __rte_unused,
4614                     char **tokens,
4615                     int n_tokens,
4616                     struct instruction *instr,
4617                     struct instruction_data *data)
4618 {
4619         CHECK(n_tokens == 2, EINVAL);
4620
4621         strcpy(data->jmp_label, tokens[1]);
4622
4623         instr->type = INSTR_JMP;
4624         instr->jmp.ip = NULL; /* Resolved later. */
4625         return 0;
4626 }
4627
4628 static int
4629 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4630                           struct action *action __rte_unused,
4631                           char **tokens,
4632                           int n_tokens,
4633                           struct instruction *instr,
4634                           struct instruction_data *data)
4635 {
4636         struct header *h;
4637
4638         CHECK(n_tokens == 3, EINVAL);
4639
4640         strcpy(data->jmp_label, tokens[1]);
4641
4642         h = header_parse(p, tokens[2]);
4643         CHECK(h, EINVAL);
4644
4645         instr->type = INSTR_JMP_VALID;
4646         instr->jmp.ip = NULL; /* Resolved later. */
4647         instr->jmp.header_id = h->id;
4648         return 0;
4649 }
4650
4651 static int
4652 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4653                             struct action *action __rte_unused,
4654                             char **tokens,
4655                             int n_tokens,
4656                             struct instruction *instr,
4657                             struct instruction_data *data)
4658 {
4659         struct header *h;
4660
4661         CHECK(n_tokens == 2, EINVAL);
4662
4663         strcpy(data->jmp_label, tokens[1]);
4664
4665         h = header_parse(p, tokens[2]);
4666         CHECK(h, EINVAL);
4667
4668         instr->type = INSTR_JMP_INVALID;
4669         instr->jmp.ip = NULL; /* Resolved later. */
4670         instr->jmp.header_id = h->id;
4671         return 0;
4672 }
4673
4674 static int
4675 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4676                         struct action *action,
4677                         char **tokens,
4678                         int n_tokens,
4679                         struct instruction *instr,
4680                         struct instruction_data *data)
4681 {
4682         CHECK(!action, EINVAL);
4683         CHECK(n_tokens == 2, EINVAL);
4684
4685         strcpy(data->jmp_label, tokens[1]);
4686
4687         instr->type = INSTR_JMP_HIT;
4688         instr->jmp.ip = NULL; /* Resolved later. */
4689         return 0;
4690 }
4691
4692 static int
4693 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4694                          struct action *action,
4695                          char **tokens,
4696                          int n_tokens,
4697                          struct instruction *instr,
4698                          struct instruction_data *data)
4699 {
4700         CHECK(!action, EINVAL);
4701         CHECK(n_tokens == 2, EINVAL);
4702
4703         strcpy(data->jmp_label, tokens[1]);
4704
4705         instr->type = INSTR_JMP_MISS;
4706         instr->jmp.ip = NULL; /* Resolved later. */
4707         return 0;
4708 }
4709
4710 static int
4711 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4712                                struct action *action,
4713                                char **tokens,
4714                                int n_tokens,
4715                                struct instruction *instr,
4716                                struct instruction_data *data)
4717 {
4718         struct action *a;
4719
4720         CHECK(!action, EINVAL);
4721         CHECK(n_tokens == 3, EINVAL);
4722
4723         strcpy(data->jmp_label, tokens[1]);
4724
4725         a = action_find(p, tokens[2]);
4726         CHECK(a, EINVAL);
4727
4728         instr->type = INSTR_JMP_ACTION_HIT;
4729         instr->jmp.ip = NULL; /* Resolved later. */
4730         instr->jmp.action_id = a->id;
4731         return 0;
4732 }
4733
4734 static int
4735 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4736                                 struct action *action,
4737                                 char **tokens,
4738                                 int n_tokens,
4739                                 struct instruction *instr,
4740                                 struct instruction_data *data)
4741 {
4742         struct action *a;
4743
4744         CHECK(!action, EINVAL);
4745         CHECK(n_tokens == 3, EINVAL);
4746
4747         strcpy(data->jmp_label, tokens[1]);
4748
4749         a = action_find(p, tokens[2]);
4750         CHECK(a, EINVAL);
4751
4752         instr->type = INSTR_JMP_ACTION_MISS;
4753         instr->jmp.ip = NULL; /* Resolved later. */
4754         instr->jmp.action_id = a->id;
4755         return 0;
4756 }
4757
4758 static int
4759 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
4760                        struct action *action,
4761                        char **tokens,
4762                        int n_tokens,
4763                        struct instruction *instr,
4764                        struct instruction_data *data)
4765 {
4766         char *a = tokens[2], *b = tokens[3];
4767         struct field *fa, *fb;
4768         uint32_t a_struct_id, b_struct_id, b_val;
4769
4770         CHECK(n_tokens == 4, EINVAL);
4771
4772         strcpy(data->jmp_label, tokens[1]);
4773
4774         fa = struct_field_parse(p, action, a, &a_struct_id);
4775         CHECK(fa, EINVAL);
4776
4777         /* JMP_EQ or JMP_EQ_S. */
4778         fb = struct_field_parse(p, action, b, &b_struct_id);
4779         if (fb) {
4780                 instr->type = INSTR_JMP_EQ;
4781                 if ((a[0] == 'h' && b[0] != 'h') ||
4782                     (a[0] != 'h' && b[0] == 'h'))
4783                         instr->type = INSTR_JMP_EQ_S;
4784                 instr->jmp.ip = NULL; /* Resolved later. */
4785
4786                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4787                 instr->jmp.a.n_bits = fa->n_bits;
4788                 instr->jmp.a.offset = fa->offset / 8;
4789                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4790                 instr->jmp.b.n_bits = fb->n_bits;
4791                 instr->jmp.b.offset = fb->offset / 8;
4792                 return 0;
4793         }
4794
4795         /* JMP_EQ_I. */
4796         b_val = strtoul(b, &b, 0);
4797         CHECK(!b[0], EINVAL);
4798
4799         if (a[0] == 'h')
4800                 b_val = htonl(b_val);
4801
4802         instr->type = INSTR_JMP_EQ_I;
4803         instr->jmp.ip = NULL; /* Resolved later. */
4804         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4805         instr->jmp.a.n_bits = fa->n_bits;
4806         instr->jmp.a.offset = fa->offset / 8;
4807         instr->jmp.b_val = (uint32_t)b_val;
4808         return 0;
4809 }
4810
4811 static int
4812 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
4813                         struct action *action,
4814                         char **tokens,
4815                         int n_tokens,
4816                         struct instruction *instr,
4817                         struct instruction_data *data)
4818 {
4819         char *a = tokens[2], *b = tokens[3];
4820         struct field *fa, *fb;
4821         uint32_t a_struct_id, b_struct_id, b_val;
4822
4823         CHECK(n_tokens == 4, EINVAL);
4824
4825         strcpy(data->jmp_label, tokens[1]);
4826
4827         fa = struct_field_parse(p, action, a, &a_struct_id);
4828         CHECK(fa, EINVAL);
4829
4830         /* JMP_NEQ or JMP_NEQ_S. */
4831         fb = struct_field_parse(p, action, b, &b_struct_id);
4832         if (fb) {
4833                 instr->type = INSTR_JMP_NEQ;
4834                 if ((a[0] == 'h' && b[0] != 'h') ||
4835                     (a[0] != 'h' && b[0] == 'h'))
4836                         instr->type = INSTR_JMP_NEQ_S;
4837                 instr->jmp.ip = NULL; /* Resolved later. */
4838
4839                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4840                 instr->jmp.a.n_bits = fa->n_bits;
4841                 instr->jmp.a.offset = fa->offset / 8;
4842                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4843                 instr->jmp.b.n_bits = fb->n_bits;
4844                 instr->jmp.b.offset = fb->offset / 8;
4845                 return 0;
4846         }
4847
4848         /* JMP_NEQ_I. */
4849         b_val = strtoul(b, &b, 0);
4850         CHECK(!b[0], EINVAL);
4851
4852         if (a[0] == 'h')
4853                 b_val = htonl(b_val);
4854
4855         instr->type = INSTR_JMP_NEQ_I;
4856         instr->jmp.ip = NULL; /* Resolved later. */
4857         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4858         instr->jmp.a.n_bits = fa->n_bits;
4859         instr->jmp.a.offset = fa->offset / 8;
4860         instr->jmp.b_val = (uint32_t)b_val;
4861         return 0;
4862 }
4863
4864 static int
4865 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
4866                        struct action *action,
4867                        char **tokens,
4868                        int n_tokens,
4869                        struct instruction *instr,
4870                        struct instruction_data *data)
4871 {
4872         char *a = tokens[2], *b = tokens[3];
4873         struct field *fa, *fb;
4874         uint32_t a_struct_id, b_struct_id, b_val;
4875
4876         CHECK(n_tokens == 4, EINVAL);
4877
4878         strcpy(data->jmp_label, tokens[1]);
4879
4880         fa = struct_field_parse(p, action, a, &a_struct_id);
4881         CHECK(fa, EINVAL);
4882
4883         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
4884         fb = struct_field_parse(p, action, b, &b_struct_id);
4885         if (fb) {
4886                 instr->type = INSTR_JMP_LT;
4887                 if (a[0] == 'h' && b[0] == 'm')
4888                         instr->type = INSTR_JMP_LT_HM;
4889                 if (a[0] == 'm' && b[0] == 'h')
4890                         instr->type = INSTR_JMP_LT_MH;
4891                 if (a[0] == 'h' && b[0] == 'h')
4892                         instr->type = INSTR_JMP_LT_HH;
4893                 instr->jmp.ip = NULL; /* Resolved later. */
4894
4895                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4896                 instr->jmp.a.n_bits = fa->n_bits;
4897                 instr->jmp.a.offset = fa->offset / 8;
4898                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4899                 instr->jmp.b.n_bits = fb->n_bits;
4900                 instr->jmp.b.offset = fb->offset / 8;
4901                 return 0;
4902         }
4903
4904         /* JMP_LT_MI, JMP_LT_HI. */
4905         b_val = strtoul(b, &b, 0);
4906         CHECK(!b[0], EINVAL);
4907
4908         instr->type = INSTR_JMP_LT_MI;
4909         if (a[0] == 'h')
4910                 instr->type = INSTR_JMP_LT_HI;
4911         instr->jmp.ip = NULL; /* Resolved later. */
4912
4913         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4914         instr->jmp.a.n_bits = fa->n_bits;
4915         instr->jmp.a.offset = fa->offset / 8;
4916         instr->jmp.b_val = (uint32_t)b_val;
4917         return 0;
4918 }
4919
4920 static int
4921 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
4922                        struct action *action,
4923                        char **tokens,
4924                        int n_tokens,
4925                        struct instruction *instr,
4926                        struct instruction_data *data)
4927 {
4928         char *a = tokens[2], *b = tokens[3];
4929         struct field *fa, *fb;
4930         uint32_t a_struct_id, b_struct_id, b_val;
4931
4932         CHECK(n_tokens == 4, EINVAL);
4933
4934         strcpy(data->jmp_label, tokens[1]);
4935
4936         fa = struct_field_parse(p, action, a, &a_struct_id);
4937         CHECK(fa, EINVAL);
4938
4939         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
4940         fb = struct_field_parse(p, action, b, &b_struct_id);
4941         if (fb) {
4942                 instr->type = INSTR_JMP_GT;
4943                 if (a[0] == 'h' && b[0] == 'm')
4944                         instr->type = INSTR_JMP_GT_HM;
4945                 if (a[0] == 'm' && b[0] == 'h')
4946                         instr->type = INSTR_JMP_GT_MH;
4947                 if (a[0] == 'h' && b[0] == 'h')
4948                         instr->type = INSTR_JMP_GT_HH;
4949                 instr->jmp.ip = NULL; /* Resolved later. */
4950
4951                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4952                 instr->jmp.a.n_bits = fa->n_bits;
4953                 instr->jmp.a.offset = fa->offset / 8;
4954                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4955                 instr->jmp.b.n_bits = fb->n_bits;
4956                 instr->jmp.b.offset = fb->offset / 8;
4957                 return 0;
4958         }
4959
4960         /* JMP_GT_MI, JMP_GT_HI. */
4961         b_val = strtoul(b, &b, 0);
4962         CHECK(!b[0], EINVAL);
4963
4964         instr->type = INSTR_JMP_GT_MI;
4965         if (a[0] == 'h')
4966                 instr->type = INSTR_JMP_GT_HI;
4967         instr->jmp.ip = NULL; /* Resolved later. */
4968
4969         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4970         instr->jmp.a.n_bits = fa->n_bits;
4971         instr->jmp.a.offset = fa->offset / 8;
4972         instr->jmp.b_val = (uint32_t)b_val;
4973         return 0;
4974 }
4975
4976 static inline void
4977 instr_jmp_exec(struct rte_swx_pipeline *p)
4978 {
4979         struct thread *t = &p->threads[p->thread_id];
4980         struct instruction *ip = t->ip;
4981
4982         TRACE("[Thread %2u] jmp\n", p->thread_id);
4983
4984         thread_ip_set(t, ip->jmp.ip);
4985 }
4986
4987 static inline void
4988 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
4989 {
4990         struct thread *t = &p->threads[p->thread_id];
4991         struct instruction *ip = t->ip;
4992         uint32_t header_id = ip->jmp.header_id;
4993
4994         TRACE("[Thread %2u] jmpv\n", p->thread_id);
4995
4996         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
4997 }
4998
4999 static inline void
5000 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5001 {
5002         struct thread *t = &p->threads[p->thread_id];
5003         struct instruction *ip = t->ip;
5004         uint32_t header_id = ip->jmp.header_id;
5005
5006         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5007
5008         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5009 }
5010
5011 static inline void
5012 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5013 {
5014         struct thread *t = &p->threads[p->thread_id];
5015         struct instruction *ip = t->ip;
5016         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5017
5018         TRACE("[Thread %2u] jmph\n", p->thread_id);
5019
5020         t->ip = ip_next[t->hit];
5021 }
5022
5023 static inline void
5024 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5025 {
5026         struct thread *t = &p->threads[p->thread_id];
5027         struct instruction *ip = t->ip;
5028         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5029
5030         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5031
5032         t->ip = ip_next[t->hit];
5033 }
5034
5035 static inline void
5036 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5037 {
5038         struct thread *t = &p->threads[p->thread_id];
5039         struct instruction *ip = t->ip;
5040
5041         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5042
5043         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5044 }
5045
5046 static inline void
5047 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5048 {
5049         struct thread *t = &p->threads[p->thread_id];
5050         struct instruction *ip = t->ip;
5051
5052         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5053
5054         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5055 }
5056
5057 static inline void
5058 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5059 {
5060         struct thread *t = &p->threads[p->thread_id];
5061         struct instruction *ip = t->ip;
5062
5063         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5064
5065         JMP_CMP(t, ip, ==);
5066 }
5067
5068 static inline void
5069 instr_jmp_eq_s_exec(struct rte_swx_pipeline *p)
5070 {
5071         struct thread *t = &p->threads[p->thread_id];
5072         struct instruction *ip = t->ip;
5073
5074         TRACE("[Thread %2u] jmpeq (s)\n", p->thread_id);
5075
5076         JMP_CMP_S(t, ip, ==);
5077 }
5078
5079 static inline void
5080 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5081 {
5082         struct thread *t = &p->threads[p->thread_id];
5083         struct instruction *ip = t->ip;
5084
5085         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5086
5087         JMP_CMP_I(t, ip, ==);
5088 }
5089
5090 static inline void
5091 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5092 {
5093         struct thread *t = &p->threads[p->thread_id];
5094         struct instruction *ip = t->ip;
5095
5096         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5097
5098         JMP_CMP(t, ip, !=);
5099 }
5100
5101 static inline void
5102 instr_jmp_neq_s_exec(struct rte_swx_pipeline *p)
5103 {
5104         struct thread *t = &p->threads[p->thread_id];
5105         struct instruction *ip = t->ip;
5106
5107         TRACE("[Thread %2u] jmpneq (s)\n", p->thread_id);
5108
5109         JMP_CMP_S(t, ip, !=);
5110 }
5111
5112 static inline void
5113 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5114 {
5115         struct thread *t = &p->threads[p->thread_id];
5116         struct instruction *ip = t->ip;
5117
5118         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5119
5120         JMP_CMP_I(t, ip, !=);
5121 }
5122
5123 static inline void
5124 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5125 {
5126         struct thread *t = &p->threads[p->thread_id];
5127         struct instruction *ip = t->ip;
5128
5129         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5130
5131         JMP_CMP(t, ip, <);
5132 }
5133
5134 static inline void
5135 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5136 {
5137         struct thread *t = &p->threads[p->thread_id];
5138         struct instruction *ip = t->ip;
5139
5140         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5141
5142         JMP_CMP_MH(t, ip, <);
5143 }
5144
5145 static inline void
5146 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5147 {
5148         struct thread *t = &p->threads[p->thread_id];
5149         struct instruction *ip = t->ip;
5150
5151         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5152
5153         JMP_CMP_HM(t, ip, <);
5154 }
5155
5156 static inline void
5157 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5158 {
5159         struct thread *t = &p->threads[p->thread_id];
5160         struct instruction *ip = t->ip;
5161
5162         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5163
5164         JMP_CMP_HH(t, ip, <);
5165 }
5166
5167 static inline void
5168 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5169 {
5170         struct thread *t = &p->threads[p->thread_id];
5171         struct instruction *ip = t->ip;
5172
5173         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5174
5175         JMP_CMP_MI(t, ip, <);
5176 }
5177
5178 static inline void
5179 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5180 {
5181         struct thread *t = &p->threads[p->thread_id];
5182         struct instruction *ip = t->ip;
5183
5184         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5185
5186         JMP_CMP_HI(t, ip, <);
5187 }
5188
5189 static inline void
5190 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5191 {
5192         struct thread *t = &p->threads[p->thread_id];
5193         struct instruction *ip = t->ip;
5194
5195         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5196
5197         JMP_CMP(t, ip, >);
5198 }
5199
5200 static inline void
5201 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5202 {
5203         struct thread *t = &p->threads[p->thread_id];
5204         struct instruction *ip = t->ip;
5205
5206         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5207
5208         JMP_CMP_MH(t, ip, >);
5209 }
5210
5211 static inline void
5212 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5213 {
5214         struct thread *t = &p->threads[p->thread_id];
5215         struct instruction *ip = t->ip;
5216
5217         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5218
5219         JMP_CMP_HM(t, ip, >);
5220 }
5221
5222 static inline void
5223 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5224 {
5225         struct thread *t = &p->threads[p->thread_id];
5226         struct instruction *ip = t->ip;
5227
5228         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5229
5230         JMP_CMP_HH(t, ip, >);
5231 }
5232
5233 static inline void
5234 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5235 {
5236         struct thread *t = &p->threads[p->thread_id];
5237         struct instruction *ip = t->ip;
5238
5239         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5240
5241         JMP_CMP_MI(t, ip, >);
5242 }
5243
5244 static inline void
5245 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5246 {
5247         struct thread *t = &p->threads[p->thread_id];
5248         struct instruction *ip = t->ip;
5249
5250         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5251
5252         JMP_CMP_HI(t, ip, >);
5253 }
5254
5255 /*
5256  * return.
5257  */
5258 static int
5259 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5260                        struct action *action,
5261                        char **tokens __rte_unused,
5262                        int n_tokens,
5263                        struct instruction *instr,
5264                        struct instruction_data *data __rte_unused)
5265 {
5266         CHECK(action, EINVAL);
5267         CHECK(n_tokens == 1, EINVAL);
5268
5269         instr->type = INSTR_RETURN;
5270         return 0;
5271 }
5272
5273 static inline void
5274 instr_return_exec(struct rte_swx_pipeline *p)
5275 {
5276         struct thread *t = &p->threads[p->thread_id];
5277
5278         TRACE("[Thread %2u] return\n", p->thread_id);
5279
5280         t->ip = t->ret;
5281 }
5282
5283 #define RTE_SWX_INSTRUCTION_TOKENS_MAX 16
5284
5285 static int
5286 instr_translate(struct rte_swx_pipeline *p,
5287                 struct action *action,
5288                 char *string,
5289                 struct instruction *instr,
5290                 struct instruction_data *data)
5291 {
5292         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5293         int n_tokens = 0, tpos = 0;
5294
5295         /* Parse the instruction string into tokens. */
5296         for ( ; ; ) {
5297                 char *token;
5298
5299                 token = strtok_r(string, " \t\v", &string);
5300                 if (!token)
5301                         break;
5302
5303                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5304
5305                 tokens[n_tokens] = token;
5306                 n_tokens++;
5307         }
5308
5309         CHECK(n_tokens, EINVAL);
5310
5311         /* Handle the optional instruction label. */
5312         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5313                 strcpy(data->label, tokens[0]);
5314
5315                 tpos += 2;
5316                 CHECK(n_tokens - tpos, EINVAL);
5317         }
5318
5319         /* Identify the instruction type. */
5320         if (!strcmp(tokens[tpos], "rx"))
5321                 return instr_rx_translate(p,
5322                                           action,
5323                                           &tokens[tpos],
5324                                           n_tokens - tpos,
5325                                           instr,
5326                                           data);
5327
5328         if (!strcmp(tokens[tpos], "tx"))
5329                 return instr_tx_translate(p,
5330                                           action,
5331                                           &tokens[tpos],
5332                                           n_tokens - tpos,
5333                                           instr,
5334                                           data);
5335
5336         if (!strcmp(tokens[tpos], "extract"))
5337                 return instr_hdr_extract_translate(p,
5338                                                    action,
5339                                                    &tokens[tpos],
5340                                                    n_tokens - tpos,
5341                                                    instr,
5342                                                    data);
5343
5344         if (!strcmp(tokens[tpos], "emit"))
5345                 return instr_hdr_emit_translate(p,
5346                                                 action,
5347                                                 &tokens[tpos],
5348                                                 n_tokens - tpos,
5349                                                 instr,
5350                                                 data);
5351
5352         if (!strcmp(tokens[tpos], "validate"))
5353                 return instr_hdr_validate_translate(p,
5354                                                     action,
5355                                                     &tokens[tpos],
5356                                                     n_tokens - tpos,
5357                                                     instr,
5358                                                     data);
5359
5360         if (!strcmp(tokens[tpos], "invalidate"))
5361                 return instr_hdr_invalidate_translate(p,
5362                                                       action,
5363                                                       &tokens[tpos],
5364                                                       n_tokens - tpos,
5365                                                       instr,
5366                                                       data);
5367
5368         if (!strcmp(tokens[tpos], "mov"))
5369                 return instr_mov_translate(p,
5370                                            action,
5371                                            &tokens[tpos],
5372                                            n_tokens - tpos,
5373                                            instr,
5374                                            data);
5375
5376         if (!strcmp(tokens[tpos], "dma"))
5377                 return instr_dma_translate(p,
5378                                            action,
5379                                            &tokens[tpos],
5380                                            n_tokens - tpos,
5381                                            instr,
5382                                            data);
5383
5384         if (!strcmp(tokens[tpos], "add"))
5385                 return instr_alu_add_translate(p,
5386                                                action,
5387                                                &tokens[tpos],
5388                                                n_tokens - tpos,
5389                                                instr,
5390                                                data);
5391
5392         if (!strcmp(tokens[tpos], "sub"))
5393                 return instr_alu_sub_translate(p,
5394                                                action,
5395                                                &tokens[tpos],
5396                                                n_tokens - tpos,
5397                                                instr,
5398                                                data);
5399
5400         if (!strcmp(tokens[tpos], "ckadd"))
5401                 return instr_alu_ckadd_translate(p,
5402                                                  action,
5403                                                  &tokens[tpos],
5404                                                  n_tokens - tpos,
5405                                                  instr,
5406                                                  data);
5407
5408         if (!strcmp(tokens[tpos], "cksub"))
5409                 return instr_alu_cksub_translate(p,
5410                                                  action,
5411                                                  &tokens[tpos],
5412                                                  n_tokens - tpos,
5413                                                  instr,
5414                                                  data);
5415
5416         if (!strcmp(tokens[tpos], "and"))
5417                 return instr_alu_and_translate(p,
5418                                                action,
5419                                                &tokens[tpos],
5420                                                n_tokens - tpos,
5421                                                instr,
5422                                                data);
5423
5424         if (!strcmp(tokens[tpos], "or"))
5425                 return instr_alu_or_translate(p,
5426                                               action,
5427                                               &tokens[tpos],
5428                                               n_tokens - tpos,
5429                                               instr,
5430                                               data);
5431
5432         if (!strcmp(tokens[tpos], "xor"))
5433                 return instr_alu_xor_translate(p,
5434                                                action,
5435                                                &tokens[tpos],
5436                                                n_tokens - tpos,
5437                                                instr,
5438                                                data);
5439
5440         if (!strcmp(tokens[tpos], "shl"))
5441                 return instr_alu_shl_translate(p,
5442                                                action,
5443                                                &tokens[tpos],
5444                                                n_tokens - tpos,
5445                                                instr,
5446                                                data);
5447
5448         if (!strcmp(tokens[tpos], "shr"))
5449                 return instr_alu_shr_translate(p,
5450                                                action,
5451                                                &tokens[tpos],
5452                                                n_tokens - tpos,
5453                                                instr,
5454                                                data);
5455
5456         if (!strcmp(tokens[tpos], "table"))
5457                 return instr_table_translate(p,
5458                                              action,
5459                                              &tokens[tpos],
5460                                              n_tokens - tpos,
5461                                              instr,
5462                                              data);
5463
5464         if (!strcmp(tokens[tpos], "extern"))
5465                 return instr_extern_translate(p,
5466                                               action,
5467                                               &tokens[tpos],
5468                                               n_tokens - tpos,
5469                                               instr,
5470                                               data);
5471
5472         if (!strcmp(tokens[tpos], "jmp"))
5473                 return instr_jmp_translate(p,
5474                                            action,
5475                                            &tokens[tpos],
5476                                            n_tokens - tpos,
5477                                            instr,
5478                                            data);
5479
5480         if (!strcmp(tokens[tpos], "jmpv"))
5481                 return instr_jmp_valid_translate(p,
5482                                                  action,
5483                                                  &tokens[tpos],
5484                                                  n_tokens - tpos,
5485                                                  instr,
5486                                                  data);
5487
5488         if (!strcmp(tokens[tpos], "jmpnv"))
5489                 return instr_jmp_invalid_translate(p,
5490                                                    action,
5491                                                    &tokens[tpos],
5492                                                    n_tokens - tpos,
5493                                                    instr,
5494                                                    data);
5495
5496         if (!strcmp(tokens[tpos], "jmph"))
5497                 return instr_jmp_hit_translate(p,
5498                                                action,
5499                                                &tokens[tpos],
5500                                                n_tokens - tpos,
5501                                                instr,
5502                                                data);
5503
5504         if (!strcmp(tokens[tpos], "jmpnh"))
5505                 return instr_jmp_miss_translate(p,
5506                                                 action,
5507                                                 &tokens[tpos],
5508                                                 n_tokens - tpos,
5509                                                 instr,
5510                                                 data);
5511
5512         if (!strcmp(tokens[tpos], "jmpa"))
5513                 return instr_jmp_action_hit_translate(p,
5514                                                       action,
5515                                                       &tokens[tpos],
5516                                                       n_tokens - tpos,
5517                                                       instr,
5518                                                       data);
5519
5520         if (!strcmp(tokens[tpos], "jmpna"))
5521                 return instr_jmp_action_miss_translate(p,
5522                                                        action,
5523                                                        &tokens[tpos],
5524                                                        n_tokens - tpos,
5525                                                        instr,
5526                                                        data);
5527
5528         if (!strcmp(tokens[tpos], "jmpeq"))
5529                 return instr_jmp_eq_translate(p,
5530                                               action,
5531                                               &tokens[tpos],
5532                                               n_tokens - tpos,
5533                                               instr,
5534                                               data);
5535
5536         if (!strcmp(tokens[tpos], "jmpneq"))
5537                 return instr_jmp_neq_translate(p,
5538                                                action,
5539                                                &tokens[tpos],
5540                                                n_tokens - tpos,
5541                                                instr,
5542                                                data);
5543
5544         if (!strcmp(tokens[tpos], "jmplt"))
5545                 return instr_jmp_lt_translate(p,
5546                                               action,
5547                                               &tokens[tpos],
5548                                               n_tokens - tpos,
5549                                               instr,
5550                                               data);
5551
5552         if (!strcmp(tokens[tpos], "jmpgt"))
5553                 return instr_jmp_gt_translate(p,
5554                                               action,
5555                                               &tokens[tpos],
5556                                               n_tokens - tpos,
5557                                               instr,
5558                                               data);
5559
5560         if (!strcmp(tokens[tpos], "return"))
5561                 return instr_return_translate(p,
5562                                               action,
5563                                               &tokens[tpos],
5564                                               n_tokens - tpos,
5565                                               instr,
5566                                               data);
5567
5568         CHECK(0, EINVAL);
5569 }
5570
5571 static struct instruction_data *
5572 label_find(struct instruction_data *data, uint32_t n, const char *label)
5573 {
5574         uint32_t i;
5575
5576         for (i = 0; i < n; i++)
5577                 if (!strcmp(label, data[i].label))
5578                         return &data[i];
5579
5580         return NULL;
5581 }
5582
5583 static uint32_t
5584 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5585 {
5586         uint32_t count = 0, i;
5587
5588         if (!label[0])
5589                 return 0;
5590
5591         for (i = 0; i < n; i++)
5592                 if (!strcmp(label, data[i].jmp_label))
5593                         count++;
5594
5595         return count;
5596 }
5597
5598 static int
5599 instr_label_check(struct instruction_data *instruction_data,
5600                   uint32_t n_instructions)
5601 {
5602         uint32_t i;
5603
5604         /* Check that all instruction labels are unique. */
5605         for (i = 0; i < n_instructions; i++) {
5606                 struct instruction_data *data = &instruction_data[i];
5607                 char *label = data->label;
5608                 uint32_t j;
5609
5610                 if (!label[0])
5611                         continue;
5612
5613                 for (j = i + 1; j < n_instructions; j++)
5614                         CHECK(strcmp(label, data[j].label), EINVAL);
5615         }
5616
5617         /* Get users for each instruction label. */
5618         for (i = 0; i < n_instructions; i++) {
5619                 struct instruction_data *data = &instruction_data[i];
5620                 char *label = data->label;
5621
5622                 data->n_users = label_is_used(instruction_data,
5623                                               n_instructions,
5624                                               label);
5625         }
5626
5627         return 0;
5628 }
5629
5630 static int
5631 instr_jmp_resolve(struct instruction *instructions,
5632                   struct instruction_data *instruction_data,
5633                   uint32_t n_instructions)
5634 {
5635         uint32_t i;
5636
5637         for (i = 0; i < n_instructions; i++) {
5638                 struct instruction *instr = &instructions[i];
5639                 struct instruction_data *data = &instruction_data[i];
5640                 struct instruction_data *found;
5641
5642                 if (!instruction_is_jmp(instr))
5643                         continue;
5644
5645                 found = label_find(instruction_data,
5646                                    n_instructions,
5647                                    data->jmp_label);
5648                 CHECK(found, EINVAL);
5649
5650                 instr->jmp.ip = &instr[found - instruction_data];
5651         }
5652
5653         return 0;
5654 }
5655
5656 static int
5657 instr_verify(struct rte_swx_pipeline *p __rte_unused,
5658              struct action *a,
5659              struct instruction *instr,
5660              struct instruction_data *data __rte_unused,
5661              uint32_t n_instructions)
5662 {
5663         if (!a) {
5664                 enum instruction_type type;
5665                 uint32_t i;
5666
5667                 /* Check that the first instruction is rx. */
5668                 CHECK(instr[0].type == INSTR_RX, EINVAL);
5669
5670                 /* Check that there is at least one tx instruction. */
5671                 for (i = 0; i < n_instructions; i++) {
5672                         type = instr[i].type;
5673
5674                         if (instr[i].type == INSTR_TX)
5675                                 break;
5676                 }
5677                 CHECK(i < n_instructions, EINVAL);
5678
5679                 /* Check that the last instruction is either tx or unconditional
5680                  * jump.
5681                  */
5682                 type = instr[n_instructions - 1].type;
5683                 CHECK((type == INSTR_TX) || (type == INSTR_JMP), EINVAL);
5684         }
5685
5686         if (a) {
5687                 enum instruction_type type;
5688                 uint32_t i;
5689
5690                 /* Check that there is at least one return or tx instruction. */
5691                 for (i = 0; i < n_instructions; i++) {
5692                         type = instr[i].type;
5693
5694                         if ((type == INSTR_RETURN) || (type == INSTR_TX))
5695                                 break;
5696                 }
5697                 CHECK(i < n_instructions, EINVAL);
5698         }
5699
5700         return 0;
5701 }
5702
5703 static int
5704 instruction_config(struct rte_swx_pipeline *p,
5705                    struct action *a,
5706                    const char **instructions,
5707                    uint32_t n_instructions)
5708 {
5709         struct instruction *instr = NULL;
5710         struct instruction_data *data = NULL;
5711         char *string = NULL;
5712         int err = 0;
5713         uint32_t i;
5714
5715         CHECK(n_instructions, EINVAL);
5716         CHECK(instructions, EINVAL);
5717         for (i = 0; i < n_instructions; i++)
5718                 CHECK(instructions[i], EINVAL);
5719
5720         /* Memory allocation. */
5721         instr = calloc(n_instructions, sizeof(struct instruction));
5722         if (!instr) {
5723                 err = ENOMEM;
5724                 goto error;
5725         }
5726
5727         data = calloc(n_instructions, sizeof(struct instruction_data));
5728         if (!data) {
5729                 err = ENOMEM;
5730                 goto error;
5731         }
5732
5733         for (i = 0; i < n_instructions; i++) {
5734                 string = strdup(instructions[i]);
5735                 if (!string) {
5736                         err = ENOMEM;
5737                         goto error;
5738                 }
5739
5740                 err = instr_translate(p, a, string, &instr[i], &data[i]);
5741                 if (err)
5742                         goto error;
5743
5744                 free(string);
5745         }
5746
5747         err = instr_label_check(data, n_instructions);
5748         if (err)
5749                 goto error;
5750
5751         err = instr_verify(p, a, instr, data, n_instructions);
5752         if (err)
5753                 goto error;
5754
5755         err = instr_jmp_resolve(instr, data, n_instructions);
5756         if (err)
5757                 goto error;
5758
5759         free(data);
5760
5761         if (a) {
5762                 a->instructions = instr;
5763                 a->n_instructions = n_instructions;
5764         } else {
5765                 p->instructions = instr;
5766                 p->n_instructions = n_instructions;
5767         }
5768
5769         return 0;
5770
5771 error:
5772         free(string);
5773         free(data);
5774         free(instr);
5775         return err;
5776 }
5777
5778 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
5779
5780 static instr_exec_t instruction_table[] = {
5781         [INSTR_RX] = instr_rx_exec,
5782         [INSTR_TX] = instr_tx_exec,
5783
5784         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
5785         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
5786         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
5787         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
5788         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
5789         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
5790         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
5791         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
5792
5793         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
5794         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
5795         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
5796         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
5797         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
5798         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
5799         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
5800         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
5801         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
5802
5803         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
5804         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
5805
5806         [INSTR_MOV] = instr_mov_exec,
5807         [INSTR_MOV_S] = instr_mov_s_exec,
5808         [INSTR_MOV_I] = instr_mov_i_exec,
5809
5810         [INSTR_DMA_HT] = instr_dma_ht_exec,
5811         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
5812         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
5813         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
5814         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
5815         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
5816         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
5817         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
5818
5819         [INSTR_ALU_ADD] = instr_alu_add_exec,
5820         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
5821         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
5822         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
5823         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
5824         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
5825
5826         [INSTR_ALU_SUB] = instr_alu_sub_exec,
5827         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
5828         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
5829         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
5830         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
5831         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
5832
5833         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
5834         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
5835         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
5836         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
5837
5838         [INSTR_ALU_AND] = instr_alu_and_exec,
5839         [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
5840         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
5841
5842         [INSTR_ALU_OR] = instr_alu_or_exec,
5843         [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
5844         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
5845
5846         [INSTR_ALU_XOR] = instr_alu_xor_exec,
5847         [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec,
5848         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
5849
5850         [INSTR_ALU_SHL] = instr_alu_shl_exec,
5851         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
5852         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
5853         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
5854         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
5855         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
5856
5857         [INSTR_ALU_SHR] = instr_alu_shr_exec,
5858         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
5859         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
5860         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
5861         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
5862         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
5863
5864         [INSTR_TABLE] = instr_table_exec,
5865         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
5866         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
5867
5868         [INSTR_JMP] = instr_jmp_exec,
5869         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
5870         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
5871         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
5872         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
5873         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
5874         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
5875
5876         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
5877         [INSTR_JMP_EQ_S] = instr_jmp_eq_s_exec,
5878         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
5879
5880         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
5881         [INSTR_JMP_NEQ_S] = instr_jmp_neq_s_exec,
5882         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
5883
5884         [INSTR_JMP_LT] = instr_jmp_lt_exec,
5885         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
5886         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
5887         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
5888         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
5889         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
5890
5891         [INSTR_JMP_GT] = instr_jmp_gt_exec,
5892         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
5893         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
5894         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
5895         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
5896         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
5897
5898         [INSTR_RETURN] = instr_return_exec,
5899 };
5900
5901 static inline void
5902 instr_exec(struct rte_swx_pipeline *p)
5903 {
5904         struct thread *t = &p->threads[p->thread_id];
5905         struct instruction *ip = t->ip;
5906         instr_exec_t instr = instruction_table[ip->type];
5907
5908         instr(p);
5909 }
5910
5911 /*
5912  * Action.
5913  */
5914 static struct action *
5915 action_find(struct rte_swx_pipeline *p, const char *name)
5916 {
5917         struct action *elem;
5918
5919         if (!name)
5920                 return NULL;
5921
5922         TAILQ_FOREACH(elem, &p->actions, node)
5923                 if (strcmp(elem->name, name) == 0)
5924                         return elem;
5925
5926         return NULL;
5927 }
5928
5929 static struct field *
5930 action_field_find(struct action *a, const char *name)
5931 {
5932         return a->st ? struct_type_field_find(a->st, name) : NULL;
5933 }
5934
5935 static struct field *
5936 action_field_parse(struct action *action, const char *name)
5937 {
5938         if (name[0] != 't' || name[1] != '.')
5939                 return NULL;
5940
5941         return action_field_find(action, &name[2]);
5942 }
5943
5944 int
5945 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
5946                                const char *name,
5947                                const char *args_struct_type_name,
5948                                const char **instructions,
5949                                uint32_t n_instructions)
5950 {
5951         struct struct_type *args_struct_type;
5952         struct action *a;
5953         int err;
5954
5955         CHECK(p, EINVAL);
5956
5957         CHECK_NAME(name, EINVAL);
5958         CHECK(!action_find(p, name), EEXIST);
5959
5960         if (args_struct_type_name) {
5961                 CHECK_NAME(args_struct_type_name, EINVAL);
5962                 args_struct_type = struct_type_find(p, args_struct_type_name);
5963                 CHECK(args_struct_type, EINVAL);
5964         } else {
5965                 args_struct_type = NULL;
5966         }
5967
5968         /* Node allocation. */
5969         a = calloc(1, sizeof(struct action));
5970         CHECK(a, ENOMEM);
5971
5972         /* Node initialization. */
5973         strcpy(a->name, name);
5974         a->st = args_struct_type;
5975         a->id = p->n_actions;
5976
5977         /* Instruction translation. */
5978         err = instruction_config(p, a, instructions, n_instructions);
5979         if (err) {
5980                 free(a);
5981                 return err;
5982         }
5983
5984         /* Node add to tailq. */
5985         TAILQ_INSERT_TAIL(&p->actions, a, node);
5986         p->n_actions++;
5987
5988         return 0;
5989 }
5990
5991 static int
5992 action_build(struct rte_swx_pipeline *p)
5993 {
5994         struct action *action;
5995
5996         p->action_instructions = calloc(p->n_actions,
5997                                         sizeof(struct instruction *));
5998         CHECK(p->action_instructions, ENOMEM);
5999
6000         TAILQ_FOREACH(action, &p->actions, node)
6001                 p->action_instructions[action->id] = action->instructions;
6002
6003         return 0;
6004 }
6005
6006 static void
6007 action_build_free(struct rte_swx_pipeline *p)
6008 {
6009         free(p->action_instructions);
6010         p->action_instructions = NULL;
6011 }
6012
6013 static void
6014 action_free(struct rte_swx_pipeline *p)
6015 {
6016         action_build_free(p);
6017
6018         for ( ; ; ) {
6019                 struct action *action;
6020
6021                 action = TAILQ_FIRST(&p->actions);
6022                 if (!action)
6023                         break;
6024
6025                 TAILQ_REMOVE(&p->actions, action, node);
6026                 free(action->instructions);
6027                 free(action);
6028         }
6029 }
6030
6031 /*
6032  * Table.
6033  */
6034 static struct table_type *
6035 table_type_find(struct rte_swx_pipeline *p, const char *name)
6036 {
6037         struct table_type *elem;
6038
6039         TAILQ_FOREACH(elem, &p->table_types, node)
6040                 if (strcmp(elem->name, name) == 0)
6041                         return elem;
6042
6043         return NULL;
6044 }
6045
6046 static struct table_type *
6047 table_type_resolve(struct rte_swx_pipeline *p,
6048                    const char *recommended_type_name,
6049                    enum rte_swx_table_match_type match_type)
6050 {
6051         struct table_type *elem;
6052
6053         /* Only consider the recommended type if the match type is correct. */
6054         if (recommended_type_name)
6055                 TAILQ_FOREACH(elem, &p->table_types, node)
6056                         if (!strcmp(elem->name, recommended_type_name) &&
6057                             (elem->match_type == match_type))
6058                                 return elem;
6059
6060         /* Ignore the recommended type and get the first element with this match
6061          * type.
6062          */
6063         TAILQ_FOREACH(elem, &p->table_types, node)
6064                 if (elem->match_type == match_type)
6065                         return elem;
6066
6067         return NULL;
6068 }
6069
6070 static struct table *
6071 table_find(struct rte_swx_pipeline *p, const char *name)
6072 {
6073         struct table *elem;
6074
6075         TAILQ_FOREACH(elem, &p->tables, node)
6076                 if (strcmp(elem->name, name) == 0)
6077                         return elem;
6078
6079         return NULL;
6080 }
6081
6082 static struct table *
6083 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6084 {
6085         struct table *table = NULL;
6086
6087         TAILQ_FOREACH(table, &p->tables, node)
6088                 if (table->id == id)
6089                         return table;
6090
6091         return NULL;
6092 }
6093
6094 int
6095 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
6096                                      const char *name,
6097                                      enum rte_swx_table_match_type match_type,
6098                                      struct rte_swx_table_ops *ops)
6099 {
6100         struct table_type *elem;
6101
6102         CHECK(p, EINVAL);
6103
6104         CHECK_NAME(name, EINVAL);
6105         CHECK(!table_type_find(p, name), EEXIST);
6106
6107         CHECK(ops, EINVAL);
6108         CHECK(ops->create, EINVAL);
6109         CHECK(ops->lkp, EINVAL);
6110         CHECK(ops->free, EINVAL);
6111
6112         /* Node allocation. */
6113         elem = calloc(1, sizeof(struct table_type));
6114         CHECK(elem, ENOMEM);
6115
6116         /* Node initialization. */
6117         strcpy(elem->name, name);
6118         elem->match_type = match_type;
6119         memcpy(&elem->ops, ops, sizeof(*ops));
6120
6121         /* Node add to tailq. */
6122         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
6123
6124         return 0;
6125 }
6126
6127 static enum rte_swx_table_match_type
6128 table_match_type_resolve(struct rte_swx_match_field_params *fields,
6129                          uint32_t n_fields)
6130 {
6131         uint32_t i;
6132
6133         for (i = 0; i < n_fields; i++)
6134                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
6135                         break;
6136
6137         if (i == n_fields)
6138                 return RTE_SWX_TABLE_MATCH_EXACT;
6139
6140         if ((i == n_fields - 1) &&
6141             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
6142                 return RTE_SWX_TABLE_MATCH_LPM;
6143
6144         return RTE_SWX_TABLE_MATCH_WILDCARD;
6145 }
6146
6147 int
6148 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
6149                               const char *name,
6150                               struct rte_swx_pipeline_table_params *params,
6151                               const char *recommended_table_type_name,
6152                               const char *args,
6153                               uint32_t size)
6154 {
6155         struct table_type *type;
6156         struct table *t;
6157         struct action *default_action;
6158         struct header *header = NULL;
6159         int is_header = 0;
6160         uint32_t offset_prev = 0, action_data_size_max = 0, i;
6161
6162         CHECK(p, EINVAL);
6163
6164         CHECK_NAME(name, EINVAL);
6165         CHECK(!table_find(p, name), EEXIST);
6166
6167         CHECK(params, EINVAL);
6168
6169         /* Match checks. */
6170         CHECK(!params->n_fields || params->fields, EINVAL);
6171         for (i = 0; i < params->n_fields; i++) {
6172                 struct rte_swx_match_field_params *field = &params->fields[i];
6173                 struct header *h;
6174                 struct field *hf, *mf;
6175                 uint32_t offset;
6176
6177                 CHECK_NAME(field->name, EINVAL);
6178
6179                 hf = header_field_parse(p, field->name, &h);
6180                 mf = metadata_field_parse(p, field->name);
6181                 CHECK(hf || mf, EINVAL);
6182
6183                 offset = hf ? hf->offset : mf->offset;
6184
6185                 if (i == 0) {
6186                         is_header = hf ? 1 : 0;
6187                         header = hf ? h : NULL;
6188                         offset_prev = offset;
6189
6190                         continue;
6191                 }
6192
6193                 CHECK((is_header && hf && (h->id == header->id)) ||
6194                       (!is_header && mf), EINVAL);
6195
6196                 CHECK(offset > offset_prev, EINVAL);
6197                 offset_prev = offset;
6198         }
6199
6200         /* Action checks. */
6201         CHECK(params->n_actions, EINVAL);
6202         CHECK(params->action_names, EINVAL);
6203         for (i = 0; i < params->n_actions; i++) {
6204                 const char *action_name = params->action_names[i];
6205                 struct action *a;
6206                 uint32_t action_data_size;
6207
6208                 CHECK(action_name, EINVAL);
6209
6210                 a = action_find(p, action_name);
6211                 CHECK(a, EINVAL);
6212
6213                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
6214                 if (action_data_size > action_data_size_max)
6215                         action_data_size_max = action_data_size;
6216         }
6217
6218         CHECK(params->default_action_name, EINVAL);
6219         for (i = 0; i < p->n_actions; i++)
6220                 if (!strcmp(params->action_names[i],
6221                             params->default_action_name))
6222                         break;
6223         CHECK(i < params->n_actions, EINVAL);
6224         default_action = action_find(p, params->default_action_name);
6225         CHECK((default_action->st && params->default_action_data) ||
6226               !params->default_action_data, EINVAL);
6227
6228         /* Table type checks. */
6229         if (params->n_fields) {
6230                 enum rte_swx_table_match_type match_type;
6231
6232                 match_type = table_match_type_resolve(params->fields,
6233                                                       params->n_fields);
6234                 type = table_type_resolve(p,
6235                                           recommended_table_type_name,
6236                                           match_type);
6237                 CHECK(type, EINVAL);
6238         } else {
6239                 type = NULL;
6240         }
6241
6242         /* Memory allocation. */
6243         t = calloc(1, sizeof(struct table));
6244         CHECK(t, ENOMEM);
6245
6246         t->fields = calloc(params->n_fields, sizeof(struct match_field));
6247         if (!t->fields) {
6248                 free(t);
6249                 CHECK(0, ENOMEM);
6250         }
6251
6252         t->actions = calloc(params->n_actions, sizeof(struct action *));
6253         if (!t->actions) {
6254                 free(t->fields);
6255                 free(t);
6256                 CHECK(0, ENOMEM);
6257         }
6258
6259         if (action_data_size_max) {
6260                 t->default_action_data = calloc(1, action_data_size_max);
6261                 if (!t->default_action_data) {
6262                         free(t->actions);
6263                         free(t->fields);
6264                         free(t);
6265                         CHECK(0, ENOMEM);
6266                 }
6267         }
6268
6269         /* Node initialization. */
6270         strcpy(t->name, name);
6271         if (args && args[0])
6272                 strcpy(t->args, args);
6273         t->type = type;
6274
6275         for (i = 0; i < params->n_fields; i++) {
6276                 struct rte_swx_match_field_params *field = &params->fields[i];
6277                 struct match_field *f = &t->fields[i];
6278
6279                 f->match_type = field->match_type;
6280                 f->field = is_header ?
6281                         header_field_parse(p, field->name, NULL) :
6282                         metadata_field_parse(p, field->name);
6283         }
6284         t->n_fields = params->n_fields;
6285         t->is_header = is_header;
6286         t->header = header;
6287
6288         for (i = 0; i < params->n_actions; i++)
6289                 t->actions[i] = action_find(p, params->action_names[i]);
6290         t->default_action = default_action;
6291         if (default_action->st)
6292                 memcpy(t->default_action_data,
6293                        params->default_action_data,
6294                        default_action->st->n_bits / 8);
6295         t->n_actions = params->n_actions;
6296         t->default_action_is_const = params->default_action_is_const;
6297         t->action_data_size_max = action_data_size_max;
6298
6299         t->size = size;
6300         t->id = p->n_tables;
6301
6302         /* Node add to tailq. */
6303         TAILQ_INSERT_TAIL(&p->tables, t, node);
6304         p->n_tables++;
6305
6306         return 0;
6307 }
6308
6309 static struct rte_swx_table_params *
6310 table_params_get(struct table *table)
6311 {
6312         struct rte_swx_table_params *params;
6313         struct field *first, *last;
6314         uint8_t *key_mask;
6315         uint32_t key_size, key_offset, action_data_size, i;
6316
6317         /* Memory allocation. */
6318         params = calloc(1, sizeof(struct rte_swx_table_params));
6319         if (!params)
6320                 return NULL;
6321
6322         /* Key offset and size. */
6323         first = table->fields[0].field;
6324         last = table->fields[table->n_fields - 1].field;
6325         key_offset = first->offset / 8;
6326         key_size = (last->offset + last->n_bits - first->offset) / 8;
6327
6328         /* Memory allocation. */
6329         key_mask = calloc(1, key_size);
6330         if (!key_mask) {
6331                 free(params);
6332                 return NULL;
6333         }
6334
6335         /* Key mask. */
6336         for (i = 0; i < table->n_fields; i++) {
6337                 struct field *f = table->fields[i].field;
6338                 uint32_t start = (f->offset - first->offset) / 8;
6339                 size_t size = f->n_bits / 8;
6340
6341                 memset(&key_mask[start], 0xFF, size);
6342         }
6343
6344         /* Action data size. */
6345         action_data_size = 0;
6346         for (i = 0; i < table->n_actions; i++) {
6347                 struct action *action = table->actions[i];
6348                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
6349
6350                 if (ads > action_data_size)
6351                         action_data_size = ads;
6352         }
6353
6354         /* Fill in. */
6355         params->match_type = table->type->match_type;
6356         params->key_size = key_size;
6357         params->key_offset = key_offset;
6358         params->key_mask0 = key_mask;
6359         params->action_data_size = action_data_size;
6360         params->n_keys_max = table->size;
6361
6362         return params;
6363 }
6364
6365 static void
6366 table_params_free(struct rte_swx_table_params *params)
6367 {
6368         if (!params)
6369                 return;
6370
6371         free(params->key_mask0);
6372         free(params);
6373 }
6374
6375 static int
6376 table_state_build(struct rte_swx_pipeline *p)
6377 {
6378         struct table *table;
6379
6380         p->table_state = calloc(p->n_tables,
6381                                 sizeof(struct rte_swx_table_state));
6382         CHECK(p->table_state, ENOMEM);
6383
6384         TAILQ_FOREACH(table, &p->tables, node) {
6385                 struct rte_swx_table_state *ts = &p->table_state[table->id];
6386
6387                 if (table->type) {
6388                         struct rte_swx_table_params *params;
6389
6390                         /* ts->obj. */
6391                         params = table_params_get(table);
6392                         CHECK(params, ENOMEM);
6393
6394                         ts->obj = table->type->ops.create(params,
6395                                 NULL,
6396                                 table->args,
6397                                 p->numa_node);
6398
6399                         table_params_free(params);
6400                         CHECK(ts->obj, ENODEV);
6401                 }
6402
6403                 /* ts->default_action_data. */
6404                 if (table->action_data_size_max) {
6405                         ts->default_action_data =
6406                                 malloc(table->action_data_size_max);
6407                         CHECK(ts->default_action_data, ENOMEM);
6408
6409                         memcpy(ts->default_action_data,
6410                                table->default_action_data,
6411                                table->action_data_size_max);
6412                 }
6413
6414                 /* ts->default_action_id. */
6415                 ts->default_action_id = table->default_action->id;
6416         }
6417
6418         return 0;
6419 }
6420
6421 static void
6422 table_state_build_free(struct rte_swx_pipeline *p)
6423 {
6424         uint32_t i;
6425
6426         if (!p->table_state)
6427                 return;
6428
6429         for (i = 0; i < p->n_tables; i++) {
6430                 struct rte_swx_table_state *ts = &p->table_state[i];
6431                 struct table *table = table_find_by_id(p, i);
6432
6433                 /* ts->obj. */
6434                 if (table->type && ts->obj)
6435                         table->type->ops.free(ts->obj);
6436
6437                 /* ts->default_action_data. */
6438                 free(ts->default_action_data);
6439         }
6440
6441         free(p->table_state);
6442         p->table_state = NULL;
6443 }
6444
6445 static void
6446 table_state_free(struct rte_swx_pipeline *p)
6447 {
6448         table_state_build_free(p);
6449 }
6450
6451 static int
6452 table_stub_lkp(void *table __rte_unused,
6453                void *mailbox __rte_unused,
6454                uint8_t **key __rte_unused,
6455                uint64_t *action_id __rte_unused,
6456                uint8_t **action_data __rte_unused,
6457                int *hit)
6458 {
6459         *hit = 0;
6460         return 1; /* DONE. */
6461 }
6462
6463 static int
6464 table_build(struct rte_swx_pipeline *p)
6465 {
6466         uint32_t i;
6467
6468         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
6469                 struct thread *t = &p->threads[i];
6470                 struct table *table;
6471
6472                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
6473                 CHECK(t->tables, ENOMEM);
6474
6475                 TAILQ_FOREACH(table, &p->tables, node) {
6476                         struct table_runtime *r = &t->tables[table->id];
6477
6478                         if (table->type) {
6479                                 uint64_t size;
6480
6481                                 size = table->type->ops.mailbox_size_get();
6482
6483                                 /* r->func. */
6484                                 r->func = table->type->ops.lkp;
6485
6486                                 /* r->mailbox. */
6487                                 if (size) {
6488                                         r->mailbox = calloc(1, size);
6489                                         CHECK(r->mailbox, ENOMEM);
6490                                 }
6491
6492                                 /* r->key. */
6493                                 r->key = table->is_header ?
6494                                         &t->structs[table->header->struct_id] :
6495                                         &t->structs[p->metadata_struct_id];
6496                         } else {
6497                                 r->func = table_stub_lkp;
6498                         }
6499                 }
6500         }
6501
6502         return 0;
6503 }
6504
6505 static void
6506 table_build_free(struct rte_swx_pipeline *p)
6507 {
6508         uint32_t i;
6509
6510         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
6511                 struct thread *t = &p->threads[i];
6512                 uint32_t j;
6513
6514                 if (!t->tables)
6515                         continue;
6516
6517                 for (j = 0; j < p->n_tables; j++) {
6518                         struct table_runtime *r = &t->tables[j];
6519
6520                         free(r->mailbox);
6521                 }
6522
6523                 free(t->tables);
6524                 t->tables = NULL;
6525         }
6526 }
6527
6528 static void
6529 table_free(struct rte_swx_pipeline *p)
6530 {
6531         table_build_free(p);
6532
6533         /* Tables. */
6534         for ( ; ; ) {
6535                 struct table *elem;
6536
6537                 elem = TAILQ_FIRST(&p->tables);
6538                 if (!elem)
6539                         break;
6540
6541                 TAILQ_REMOVE(&p->tables, elem, node);
6542                 free(elem->fields);
6543                 free(elem->actions);
6544                 free(elem->default_action_data);
6545                 free(elem);
6546         }
6547
6548         /* Table types. */
6549         for ( ; ; ) {
6550                 struct table_type *elem;
6551
6552                 elem = TAILQ_FIRST(&p->table_types);
6553                 if (!elem)
6554                         break;
6555
6556                 TAILQ_REMOVE(&p->table_types, elem, node);
6557                 free(elem);
6558         }
6559 }
6560
6561 /*
6562  * Pipeline.
6563  */
6564 int
6565 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
6566 {
6567         struct rte_swx_pipeline *pipeline;
6568
6569         /* Check input parameters. */
6570         CHECK(p, EINVAL);
6571
6572         /* Memory allocation. */
6573         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
6574         CHECK(pipeline, ENOMEM);
6575
6576         /* Initialization. */
6577         TAILQ_INIT(&pipeline->struct_types);
6578         TAILQ_INIT(&pipeline->port_in_types);
6579         TAILQ_INIT(&pipeline->ports_in);
6580         TAILQ_INIT(&pipeline->port_out_types);
6581         TAILQ_INIT(&pipeline->ports_out);
6582         TAILQ_INIT(&pipeline->extern_types);
6583         TAILQ_INIT(&pipeline->extern_objs);
6584         TAILQ_INIT(&pipeline->extern_funcs);
6585         TAILQ_INIT(&pipeline->headers);
6586         TAILQ_INIT(&pipeline->actions);
6587         TAILQ_INIT(&pipeline->table_types);
6588         TAILQ_INIT(&pipeline->tables);
6589
6590         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
6591         pipeline->numa_node = numa_node;
6592
6593         *p = pipeline;
6594         return 0;
6595 }
6596
6597 void
6598 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
6599 {
6600         if (!p)
6601                 return;
6602
6603         free(p->instructions);
6604
6605         table_state_free(p);
6606         table_free(p);
6607         action_free(p);
6608         metadata_free(p);
6609         header_free(p);
6610         extern_func_free(p);
6611         extern_obj_free(p);
6612         port_out_free(p);
6613         port_in_free(p);
6614         struct_free(p);
6615
6616         free(p);
6617 }
6618
6619 int
6620 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
6621                                      const char **instructions,
6622                                      uint32_t n_instructions)
6623 {
6624         int err;
6625         uint32_t i;
6626
6627         err = instruction_config(p, NULL, instructions, n_instructions);
6628         if (err)
6629                 return err;
6630
6631         /* Thread instruction pointer reset. */
6632         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
6633                 struct thread *t = &p->threads[i];
6634
6635                 thread_ip_reset(p, t);
6636         }
6637
6638         return 0;
6639 }
6640
6641 int
6642 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
6643 {
6644         int status;
6645
6646         CHECK(p, EINVAL);
6647         CHECK(p->build_done == 0, EEXIST);
6648
6649         status = port_in_build(p);
6650         if (status)
6651                 goto error;
6652
6653         status = port_out_build(p);
6654         if (status)
6655                 goto error;
6656
6657         status = struct_build(p);
6658         if (status)
6659                 goto error;
6660
6661         status = extern_obj_build(p);
6662         if (status)
6663                 goto error;
6664
6665         status = extern_func_build(p);
6666         if (status)
6667                 goto error;
6668
6669         status = header_build(p);
6670         if (status)
6671                 goto error;
6672
6673         status = metadata_build(p);
6674         if (status)
6675                 goto error;
6676
6677         status = action_build(p);
6678         if (status)
6679                 goto error;
6680
6681         status = table_build(p);
6682         if (status)
6683                 goto error;
6684
6685         status = table_state_build(p);
6686         if (status)
6687                 goto error;
6688
6689         p->build_done = 1;
6690         return 0;
6691
6692 error:
6693         table_state_build_free(p);
6694         table_build_free(p);
6695         action_build_free(p);
6696         metadata_build_free(p);
6697         header_build_free(p);
6698         extern_func_build_free(p);
6699         extern_obj_build_free(p);
6700         port_out_build_free(p);
6701         port_in_build_free(p);
6702         struct_build_free(p);
6703
6704         return status;
6705 }
6706
6707 void
6708 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
6709 {
6710         uint32_t i;
6711
6712         for (i = 0; i < n_instructions; i++)
6713                 instr_exec(p);
6714 }
6715
6716 /*
6717  * Control.
6718  */
6719 int
6720 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
6721                                  struct rte_swx_table_state **table_state)
6722 {
6723         if (!p || !table_state || !p->build_done)
6724                 return -EINVAL;
6725
6726         *table_state = p->table_state;
6727         return 0;
6728 }
6729
6730 int
6731 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
6732                                  struct rte_swx_table_state *table_state)
6733 {
6734         if (!p || !table_state || !p->build_done)
6735                 return -EINVAL;
6736
6737         p->table_state = table_state;
6738         return 0;
6739 }