lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9 #include <arpa/inet.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_prefetch.h>
  13 #include <rte_byteorder.h>
  14
  15 #include "rte_swx_pipeline.h"
  16 #include "rte_swx_ctl.h"
  17
  18 #define CHECK(condition, err_code)                                             \
  19 do {                                                                           \
  20         if (!(condition))                                                      \
  21                 return -(err_code);                                            \
  22 } while (0)
  23
  24 #define CHECK_NAME(name, err_code)                                             \
  25         CHECK((name) && (name)[0], err_code)
  26
  27 #ifndef TRACE_LEVEL
  28 #define TRACE_LEVEL 0
  29 #endif
  30
  31 #if TRACE_LEVEL
  32 #define TRACE(...) printf(__VA_ARGS__)
  33 #else
  34 #define TRACE(...)
  35 #endif
  36
  37 #define ntoh64(x) rte_be_to_cpu_64(x)
  38 #define hton64(x) rte_cpu_to_be_64(x)
  39
  40 /*
  41  * Struct.
  42  */
  43 struct field {
  44         char name[RTE_SWX_NAME_SIZE];
  45         uint32_t n_bits;
  46         uint32_t offset;
  47 };
  48
  49 struct struct_type {
  50         TAILQ_ENTRY(struct_type) node;
  51         char name[RTE_SWX_NAME_SIZE];
  52         struct field *fields;
  53         uint32_t n_fields;
  54         uint32_t n_bits;
  55 };
  56
  57 TAILQ_HEAD(struct_type_tailq, struct_type);
  58
  59 /*
  60  * Input port.
  61  */
  62 struct port_in_type {
  63         TAILQ_ENTRY(port_in_type) node;
  64         char name[RTE_SWX_NAME_SIZE];
  65         struct rte_swx_port_in_ops ops;
  66 };
  67
  68 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  69
  70 struct port_in {
  71         TAILQ_ENTRY(port_in) node;
  72         struct port_in_type *type;
  73         void *obj;
  74         uint32_t id;
  75 };
  76
  77 TAILQ_HEAD(port_in_tailq, port_in);
  78
  79 struct port_in_runtime {
  80         rte_swx_port_in_pkt_rx_t pkt_rx;
  81         void *obj;
  82 };
  83
  84 /*
  85  * Output port.
  86  */
  87 struct port_out_type {
  88         TAILQ_ENTRY(port_out_type) node;
  89         char name[RTE_SWX_NAME_SIZE];
  90         struct rte_swx_port_out_ops ops;
  91 };
  92
  93 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  94
  95 struct port_out {
  96         TAILQ_ENTRY(port_out) node;
  97         struct port_out_type *type;
  98         void *obj;
  99         uint32_t id;
 100 };
 101
 102 TAILQ_HEAD(port_out_tailq, port_out);
 103
 104 struct port_out_runtime {
 105         rte_swx_port_out_pkt_tx_t pkt_tx;
 106         rte_swx_port_out_flush_t flush;
 107         void *obj;
 108 };
 109
 110 /*
 111  * Extern object.
 112  */
 113 struct extern_type_member_func {
 114         TAILQ_ENTRY(extern_type_member_func) node;
 115         char name[RTE_SWX_NAME_SIZE];
 116         rte_swx_extern_type_member_func_t func;
 117         uint32_t id;
 118 };
 119
 120 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 121
 122 struct extern_type {
 123         TAILQ_ENTRY(extern_type) node;
 124         char name[RTE_SWX_NAME_SIZE];
 125         struct struct_type *mailbox_struct_type;
 126         rte_swx_extern_type_constructor_t constructor;
 127         rte_swx_extern_type_destructor_t destructor;
 128         struct extern_type_member_func_tailq funcs;
 129         uint32_t n_funcs;
 130 };
 131
 132 TAILQ_HEAD(extern_type_tailq, extern_type);
 133
 134 struct extern_obj {
 135         TAILQ_ENTRY(extern_obj) node;
 136         char name[RTE_SWX_NAME_SIZE];
 137         struct extern_type *type;
 138         void *obj;
 139         uint32_t struct_id;
 140         uint32_t id;
 141 };
 142
 143 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 144
 145 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 146 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 147 #endif
 148
 149 struct extern_obj_runtime {
 150         void *obj;
 151         uint8_t *mailbox;
 152         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 153 };
 154
 155 /*
 156  * Extern function.
 157  */
 158 struct extern_func {
 159         TAILQ_ENTRY(extern_func) node;
 160         char name[RTE_SWX_NAME_SIZE];
 161         struct struct_type *mailbox_struct_type;
 162         rte_swx_extern_func_t func;
 163         uint32_t struct_id;
 164         uint32_t id;
 165 };
 166
 167 TAILQ_HEAD(extern_func_tailq, extern_func);
 168
 169 struct extern_func_runtime {
 170         uint8_t *mailbox;
 171         rte_swx_extern_func_t func;
 172 };
 173
 174 /*
 175  * Header.
 176  */
 177 struct header {
 178         TAILQ_ENTRY(header) node;
 179         char name[RTE_SWX_NAME_SIZE];
 180         struct struct_type *st;
 181         uint32_t struct_id;
 182         uint32_t id;
 183 };
 184
 185 TAILQ_HEAD(header_tailq, header);
 186
 187 struct header_runtime {
 188         uint8_t *ptr0;
 189 };
 190
 191 struct header_out_runtime {
 192         uint8_t *ptr0;
 193         uint8_t *ptr;
 194         uint32_t n_bytes;
 195 };
 196
 197 /*
 198  * Instruction.
 199  */
 200
 201 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 202  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 203  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 204  * when transferred to packet meta-data and in NBO when transferred to packet
 205  * headers.
 206  */
 207
 208 /* Notation conventions:
 209  *    -Header field: H = h.header.field (dst/src)
 210  *    -Meta-data field: M = m.field (dst/src)
 211  *    -Extern object mailbox field: E = e.field (dst/src)
 212  *    -Extern function mailbox field: F = f.field (dst/src)
 213  *    -Table action data field: T = t.field (src only)
 214  *    -Immediate value: I = 32-bit unsigned value (src only)
 215  */
 216
 217 enum instruction_type {
 218         /* rx m.port_in */
 219         INSTR_RX,
 220
 221         /* tx m.port_out */
 222         INSTR_TX,
 223
 224         /* extract h.header */
 225         INSTR_HDR_EXTRACT,
 226         INSTR_HDR_EXTRACT2,
 227         INSTR_HDR_EXTRACT3,
 228         INSTR_HDR_EXTRACT4,
 229         INSTR_HDR_EXTRACT5,
 230         INSTR_HDR_EXTRACT6,
 231         INSTR_HDR_EXTRACT7,
 232         INSTR_HDR_EXTRACT8,
 233
 234         /* emit h.header */
 235         INSTR_HDR_EMIT,
 236         INSTR_HDR_EMIT_TX,
 237         INSTR_HDR_EMIT2_TX,
 238         INSTR_HDR_EMIT3_TX,
 239         INSTR_HDR_EMIT4_TX,
 240         INSTR_HDR_EMIT5_TX,
 241         INSTR_HDR_EMIT6_TX,
 242         INSTR_HDR_EMIT7_TX,
 243         INSTR_HDR_EMIT8_TX,
 244
 245         /* validate h.header */
 246         INSTR_HDR_VALIDATE,
 247
 248         /* invalidate h.header */
 249         INSTR_HDR_INVALIDATE,
 250
 251         /* mov dst src
 252          * dst = src
 253          * dst = HMEF, src = HMEFTI
 254          */
 255         INSTR_MOV,   /* dst = MEF, src = MEFT */
 256         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 257         INSTR_MOV_I, /* dst = HMEF, src = I */
 258
 259         /* dma h.header t.field
 260          * memcpy(h.header, t.field, sizeof(h.header))
 261          */
 262         INSTR_DMA_HT,
 263         INSTR_DMA_HT2,
 264         INSTR_DMA_HT3,
 265         INSTR_DMA_HT4,
 266         INSTR_DMA_HT5,
 267         INSTR_DMA_HT6,
 268         INSTR_DMA_HT7,
 269         INSTR_DMA_HT8,
 270
 271         /* add dst src
 272          * dst += src
 273          * dst = HMEF, src = HMEFTI
 274          */
 275         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 276         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 277         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 278         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 279         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 280         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 281
 282         /* sub dst src
 283          * dst -= src
 284          * dst = HMEF, src = HMEFTI
 285          */
 286         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 287         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 288         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 289         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 290         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 291         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 292
 293         /* ckadd dst src
 294          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 295          * dst = H, src = {H, h.header}
 296          */
 297         INSTR_ALU_CKADD_FIELD,    /* src = H */
 298         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 299         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 300
 301         /* cksub dst src
 302          * dst = dst '- src
 303          * dst = H, src = H
 304          */
 305         INSTR_ALU_CKSUB_FIELD,
 306
 307         /* and dst src
 308          * dst &= src
 309          * dst = HMEF, src = HMEFTI
 310          */
 311         INSTR_ALU_AND,   /* dst = MEF, src = MEFT */
 312         INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 313         INSTR_ALU_AND_I, /* dst = HMEF, src = I */
 314
 315         /* or dst src
 316          * dst |= src
 317          * dst = HMEF, src = HMEFTI
 318          */
 319         INSTR_ALU_OR,   /* dst = MEF, src = MEFT */
 320         INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 321         INSTR_ALU_OR_I, /* dst = HMEF, src = I */
 322
 323         /* xor dst src
 324          * dst ^= src
 325          * dst = HMEF, src = HMEFTI
 326          */
 327         INSTR_ALU_XOR,   /* dst = MEF, src = MEFT */
 328         INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 329         INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
 330
 331         /* shl dst src
 332          * dst <<= src
 333          * dst = HMEF, src = HMEFTI
 334          */
 335         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 336         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 337         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 338         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 339         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 340         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 341 };
 342
 343 struct instr_operand {
 344         uint8_t struct_id;
 345         uint8_t n_bits;
 346         uint8_t offset;
 347         uint8_t pad;
 348 };
 349
 350 struct instr_io {
 351         struct {
 352                 uint8_t offset;
 353                 uint8_t n_bits;
 354                 uint8_t pad[2];
 355         } io;
 356
 357         struct {
 358                 uint8_t header_id[8];
 359                 uint8_t struct_id[8];
 360                 uint8_t n_bytes[8];
 361         } hdr;
 362 };
 363
 364 struct instr_hdr_validity {
 365         uint8_t header_id;
 366 };
 367
 368 struct instr_dst_src {
 369         struct instr_operand dst;
 370         union {
 371                 struct instr_operand src;
 372                 uint32_t src_val;
 373         };
 374 };
 375
 376 struct instr_dma {
 377         struct {
 378                 uint8_t header_id[8];
 379                 uint8_t struct_id[8];
 380         } dst;
 381
 382         struct {
 383                 uint8_t offset[8];
 384         } src;
 385
 386         uint16_t n_bytes[8];
 387 };
 388
 389 struct instruction {
 390         enum instruction_type type;
 391         union {
 392                 struct instr_io io;
 393                 struct instr_hdr_validity valid;
 394                 struct instr_dst_src mov;
 395                 struct instr_dma dma;
 396                 struct instr_dst_src alu;
 397         };
 398 };
 399
 400 struct instruction_data {
 401         char label[RTE_SWX_NAME_SIZE];
 402         char jmp_label[RTE_SWX_NAME_SIZE];
 403         uint32_t n_users; /* user = jmp instruction to this instruction. */
 404         int invalid;
 405 };
 406
 407 /*
 408  * Action.
 409  */
 410 struct action {
 411         TAILQ_ENTRY(action) node;
 412         char name[RTE_SWX_NAME_SIZE];
 413         struct struct_type *st;
 414         struct instruction *instructions;
 415         uint32_t n_instructions;
 416         uint32_t id;
 417 };
 418
 419 TAILQ_HEAD(action_tailq, action);
 420
 421 /*
 422  * Table.
 423  */
 424 struct table_type {
 425         TAILQ_ENTRY(table_type) node;
 426         char name[RTE_SWX_NAME_SIZE];
 427         enum rte_swx_table_match_type match_type;
 428         struct rte_swx_table_ops ops;
 429 };
 430
 431 TAILQ_HEAD(table_type_tailq, table_type);
 432
 433 struct match_field {
 434         enum rte_swx_table_match_type match_type;
 435         struct field *field;
 436 };
 437
 438 struct table {
 439         TAILQ_ENTRY(table) node;
 440         char name[RTE_SWX_NAME_SIZE];
 441         char args[RTE_SWX_NAME_SIZE];
 442         struct table_type *type; /* NULL when n_fields == 0. */
 443
 444         /* Match. */
 445         struct match_field *fields;
 446         uint32_t n_fields;
 447         int is_header; /* Only valid when n_fields > 0. */
 448         struct header *header; /* Only valid when n_fields > 0. */
 449
 450         /* Action. */
 451         struct action **actions;
 452         struct action *default_action;
 453         uint8_t *default_action_data;
 454         uint32_t n_actions;
 455         int default_action_is_const;
 456         uint32_t action_data_size_max;
 457
 458         uint32_t size;
 459         uint32_t id;
 460 };
 461
 462 TAILQ_HEAD(table_tailq, table);
 463
 464 struct table_runtime {
 465         rte_swx_table_lookup_t func;
 466         void *mailbox;
 467         uint8_t **key;
 468 };
 469
 470 /*
 471  * Pipeline.
 472  */
 473 struct thread {
 474         /* Packet. */
 475         struct rte_swx_pkt pkt;
 476         uint8_t *ptr;
 477
 478         /* Structures. */
 479         uint8_t **structs;
 480
 481         /* Packet headers. */
 482         struct header_runtime *headers; /* Extracted or generated headers. */
 483         struct header_out_runtime *headers_out; /* Emitted headers. */
 484         uint8_t *header_storage;
 485         uint8_t *header_out_storage;
 486         uint64_t valid_headers;
 487         uint32_t n_headers_out;
 488
 489         /* Packet meta-data. */
 490         uint8_t *metadata;
 491
 492         /* Tables. */
 493         struct table_runtime *tables;
 494         struct rte_swx_table_state *table_state;
 495         uint64_t action_id;
 496         int hit; /* 0 = Miss, 1 = Hit. */
 497
 498         /* Extern objects and functions. */
 499         struct extern_obj_runtime *extern_objs;
 500         struct extern_func_runtime *extern_funcs;
 501
 502         /* Instructions. */
 503         struct instruction *ip;
 504         struct instruction *ret;
 505 };
 506
 507 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 508 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 509 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 510
 511 #define ALU(thread, ip, operator)  \
 512 {                                                                              \
 513         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 514         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 515         uint64_t dst64 = *dst64_ptr;                                           \
 516         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 517         uint64_t dst = dst64 & dst64_mask;                                     \
 518                                                                                \
 519         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 520         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 521         uint64_t src64 = *src64_ptr;                                           \
 522         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 523         uint64_t src = src64 & src64_mask;                                     \
 524                                                                                \
 525         uint64_t result = dst operator src;                                    \
 526                                                                                \
 527         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 528 }
 529
 530 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 531
 532 #define ALU_S(thread, ip, operator)  \
 533 {                                                                              \
 534         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 535         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 536         uint64_t dst64 = *dst64_ptr;                                           \
 537         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 538         uint64_t dst = dst64 & dst64_mask;                                     \
 539                                                                                \
 540         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 541         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 542         uint64_t src64 = *src64_ptr;                                           \
 543         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 544                                                                                \
 545         uint64_t result = dst operator src;                                    \
 546                                                                                \
 547         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 548 }
 549
 550 #define ALU_MH ALU_S
 551
 552 #define ALU_HM(thread, ip, operator)  \
 553 {                                                                              \
 554         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 555         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 556         uint64_t dst64 = *dst64_ptr;                                           \
 557         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 558         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 559                                                                                \
 560         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 561         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 562         uint64_t src64 = *src64_ptr;                                           \
 563         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 564         uint64_t src = src64 & src64_mask;                                     \
 565                                                                                \
 566         uint64_t result = dst operator src;                                    \
 567         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 568                                                                                \
 569         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 570 }
 571
 572 #define ALU_HH(thread, ip, operator)  \
 573 {                                                                              \
 574         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 575         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 576         uint64_t dst64 = *dst64_ptr;                                           \
 577         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 578         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 579                                                                                \
 580         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 581         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 582         uint64_t src64 = *src64_ptr;                                           \
 583         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 584                                                                                \
 585         uint64_t result = dst operator src;                                    \
 586         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 587                                                                                \
 588         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 589 }
 590
 591 #else
 592
 593 #define ALU_S ALU
 594 #define ALU_MH ALU
 595 #define ALU_HM ALU
 596 #define ALU_HH ALU
 597
 598 #endif
 599
 600 #define ALU_I(thread, ip, operator)  \
 601 {                                                                              \
 602         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 603         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 604         uint64_t dst64 = *dst64_ptr;                                           \
 605         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 606         uint64_t dst = dst64 & dst64_mask;                                     \
 607                                                                                \
 608         uint64_t src = (ip)->alu.src_val;                                      \
 609                                                                                \
 610         uint64_t result = dst operator src;                                    \
 611                                                                                \
 612         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 613 }
 614
 615 #define ALU_MI ALU_I
 616
 617 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 618
 619 #define ALU_HI(thread, ip, operator)  \
 620 {                                                                              \
 621         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 622         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 623         uint64_t dst64 = *dst64_ptr;                                           \
 624         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 625         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 626                                                                                \
 627         uint64_t src = (ip)->alu.src_val;                                      \
 628                                                                                \
 629         uint64_t result = dst operator src;                                    \
 630         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 631                                                                                \
 632         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 633 }
 634
 635 #else
 636
 637 #define ALU_HI ALU_I
 638
 639 #endif
 640
 641 #define MOV(thread, ip)  \
 642 {                                                                              \
 643         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 644         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 645         uint64_t dst64 = *dst64_ptr;                                           \
 646         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 647                                                                                \
 648         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 649         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 650         uint64_t src64 = *src64_ptr;                                           \
 651         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
 652         uint64_t src = src64 & src64_mask;                                     \
 653                                                                                \
 654         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 655 }
 656
 657 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 658
 659 #define MOV_S(thread, ip)  \
 660 {                                                                              \
 661         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 662         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 663         uint64_t dst64 = *dst64_ptr;                                           \
 664         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 665                                                                                \
 666         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 667         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 668         uint64_t src64 = *src64_ptr;                                           \
 669         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
 670                                                                                \
 671         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 672 }
 673
 674 #else
 675
 676 #define MOV_S MOV
 677
 678 #endif
 679
 680 #define MOV_I(thread, ip)  \
 681 {                                                                              \
 682         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 683         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 684         uint64_t dst64 = *dst64_ptr;                                           \
 685         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 686                                                                                \
 687         uint64_t src = (ip)->mov.src_val;                                      \
 688                                                                                \
 689         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 690 }
 691
 692 #define METADATA_READ(thread, offset, n_bits)                                  \
 693 ({                                                                             \
 694         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 695         uint64_t m64 = *m64_ptr;                                               \
 696         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 697         (m64 & m64_mask);                                                      \
 698 })
 699
 700 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
 701 {                                                                              \
 702         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 703         uint64_t m64 = *m64_ptr;                                               \
 704         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 705                                                                                \
 706         uint64_t m_new = value;                                                \
 707                                                                                \
 708         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
 709 }
 710
 711 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 712 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 713 #endif
 714
 715 struct rte_swx_pipeline {
 716         struct struct_type_tailq struct_types;
 717         struct port_in_type_tailq port_in_types;
 718         struct port_in_tailq ports_in;
 719         struct port_out_type_tailq port_out_types;
 720         struct port_out_tailq ports_out;
 721         struct extern_type_tailq extern_types;
 722         struct extern_obj_tailq extern_objs;
 723         struct extern_func_tailq extern_funcs;
 724         struct header_tailq headers;
 725         struct struct_type *metadata_st;
 726         uint32_t metadata_struct_id;
 727         struct action_tailq actions;
 728         struct table_type_tailq table_types;
 729         struct table_tailq tables;
 730
 731         struct port_in_runtime *in;
 732         struct port_out_runtime *out;
 733         struct instruction **action_instructions;
 734         struct rte_swx_table_state *table_state;
 735         struct instruction *instructions;
 736         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 737
 738         uint32_t n_structs;
 739         uint32_t n_ports_in;
 740         uint32_t n_ports_out;
 741         uint32_t n_extern_objs;
 742         uint32_t n_extern_funcs;
 743         uint32_t n_actions;
 744         uint32_t n_tables;
 745         uint32_t n_headers;
 746         uint32_t thread_id;
 747         uint32_t port_id;
 748         uint32_t n_instructions;
 749         int build_done;
 750         int numa_node;
 751 };
 752
 753 /*
 754  * Struct.
 755  */
 756 static struct struct_type *
 757 struct_type_find(struct rte_swx_pipeline *p, const char *name)
 758 {
 759         struct struct_type *elem;
 760
 761         TAILQ_FOREACH(elem, &p->struct_types, node)
 762                 if (strcmp(elem->name, name) == 0)
 763                         return elem;
 764
 765         return NULL;
 766 }
 767
 768 static struct field *
 769 struct_type_field_find(struct struct_type *st, const char *name)
 770 {
 771         uint32_t i;
 772
 773         for (i = 0; i < st->n_fields; i++) {
 774                 struct field *f = &st->fields[i];
 775
 776                 if (strcmp(f->name, name) == 0)
 777                         return f;
 778         }
 779
 780         return NULL;
 781 }
 782
 783 int
 784 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 785                                       const char *name,
 786                                       struct rte_swx_field_params *fields,
 787                                       uint32_t n_fields)
 788 {
 789         struct struct_type *st;
 790         uint32_t i;
 791
 792         CHECK(p, EINVAL);
 793         CHECK_NAME(name, EINVAL);
 794         CHECK(fields, EINVAL);
 795         CHECK(n_fields, EINVAL);
 796
 797         for (i = 0; i < n_fields; i++) {
 798                 struct rte_swx_field_params *f = &fields[i];
 799                 uint32_t j;
 800
 801                 CHECK_NAME(f->name, EINVAL);
 802                 CHECK(f->n_bits, EINVAL);
 803                 CHECK(f->n_bits <= 64, EINVAL);
 804                 CHECK((f->n_bits & 7) == 0, EINVAL);
 805
 806                 for (j = 0; j < i; j++) {
 807                         struct rte_swx_field_params *f_prev = &fields[j];
 808
 809                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 810                 }
 811         }
 812
 813         CHECK(!struct_type_find(p, name), EEXIST);
 814
 815         /* Node allocation. */
 816         st = calloc(1, sizeof(struct struct_type));
 817         CHECK(st, ENOMEM);
 818
 819         st->fields = calloc(n_fields, sizeof(struct field));
 820         if (!st->fields) {
 821                 free(st);
 822                 CHECK(0, ENOMEM);
 823         }
 824
 825         /* Node initialization. */
 826         strcpy(st->name, name);
 827         for (i = 0; i < n_fields; i++) {
 828                 struct field *dst = &st->fields[i];
 829                 struct rte_swx_field_params *src = &fields[i];
 830
 831                 strcpy(dst->name, src->name);
 832                 dst->n_bits = src->n_bits;
 833                 dst->offset = st->n_bits;
 834
 835                 st->n_bits += src->n_bits;
 836         }
 837         st->n_fields = n_fields;
 838
 839         /* Node add to tailq. */
 840         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 841
 842         return 0;
 843 }
 844
 845 static int
 846 struct_build(struct rte_swx_pipeline *p)
 847 {
 848         uint32_t i;
 849
 850         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 851                 struct thread *t = &p->threads[i];
 852
 853                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 854                 CHECK(t->structs, ENOMEM);
 855         }
 856
 857         return 0;
 858 }
 859
 860 static void
 861 struct_build_free(struct rte_swx_pipeline *p)
 862 {
 863         uint32_t i;
 864
 865         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 866                 struct thread *t = &p->threads[i];
 867
 868                 free(t->structs);
 869                 t->structs = NULL;
 870         }
 871 }
 872
 873 static void
 874 struct_free(struct rte_swx_pipeline *p)
 875 {
 876         struct_build_free(p);
 877
 878         /* Struct types. */
 879         for ( ; ; ) {
 880                 struct struct_type *elem;
 881
 882                 elem = TAILQ_FIRST(&p->struct_types);
 883                 if (!elem)
 884                         break;
 885
 886                 TAILQ_REMOVE(&p->struct_types, elem, node);
 887                 free(elem->fields);
 888                 free(elem);
 889         }
 890 }
 891
 892 /*
 893  * Input port.
 894  */
 895 static struct port_in_type *
 896 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 897 {
 898         struct port_in_type *elem;
 899
 900         if (!name)
 901                 return NULL;
 902
 903         TAILQ_FOREACH(elem, &p->port_in_types, node)
 904                 if (strcmp(elem->name, name) == 0)
 905                         return elem;
 906
 907         return NULL;
 908 }
 909
 910 int
 911 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 912                                        const char *name,
 913                                        struct rte_swx_port_in_ops *ops)
 914 {
 915         struct port_in_type *elem;
 916
 917         CHECK(p, EINVAL);
 918         CHECK_NAME(name, EINVAL);
 919         CHECK(ops, EINVAL);
 920         CHECK(ops->create, EINVAL);
 921         CHECK(ops->free, EINVAL);
 922         CHECK(ops->pkt_rx, EINVAL);
 923         CHECK(ops->stats_read, EINVAL);
 924
 925         CHECK(!port_in_type_find(p, name), EEXIST);
 926
 927         /* Node allocation. */
 928         elem = calloc(1, sizeof(struct port_in_type));
 929         CHECK(elem, ENOMEM);
 930
 931         /* Node initialization. */
 932         strcpy(elem->name, name);
 933         memcpy(&elem->ops, ops, sizeof(*ops));
 934
 935         /* Node add to tailq. */
 936         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 937
 938         return 0;
 939 }
 940
 941 static struct port_in *
 942 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 943 {
 944         struct port_in *port;
 945
 946         TAILQ_FOREACH(port, &p->ports_in, node)
 947                 if (port->id == port_id)
 948                         return port;
 949
 950         return NULL;
 951 }
 952
 953 int
 954 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 955                                 uint32_t port_id,
 956                                 const char *port_type_name,
 957                                 void *args)
 958 {
 959         struct port_in_type *type = NULL;
 960         struct port_in *port = NULL;
 961         void *obj = NULL;
 962
 963         CHECK(p, EINVAL);
 964
 965         CHECK(!port_in_find(p, port_id), EINVAL);
 966
 967         CHECK_NAME(port_type_name, EINVAL);
 968         type = port_in_type_find(p, port_type_name);
 969         CHECK(type, EINVAL);
 970
 971         obj = type->ops.create(args);
 972         CHECK(obj, ENODEV);
 973
 974         /* Node allocation. */
 975         port = calloc(1, sizeof(struct port_in));
 976         CHECK(port, ENOMEM);
 977
 978         /* Node initialization. */
 979         port->type = type;
 980         port->obj = obj;
 981         port->id = port_id;
 982
 983         /* Node add to tailq. */
 984         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 985         if (p->n_ports_in < port_id + 1)
 986                 p->n_ports_in = port_id + 1;
 987
 988         return 0;
 989 }
 990
 991 static int
 992 port_in_build(struct rte_swx_pipeline *p)
 993 {
 994         struct port_in *port;
 995         uint32_t i;
 996
 997         CHECK(p->n_ports_in, EINVAL);
 998         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 999
1000         for (i = 0; i < p->n_ports_in; i++)
1001                 CHECK(port_in_find(p, i), EINVAL);
1002
1003         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1004         CHECK(p->in, ENOMEM);
1005
1006         TAILQ_FOREACH(port, &p->ports_in, node) {
1007                 struct port_in_runtime *in = &p->in[port->id];
1008
1009                 in->pkt_rx = port->type->ops.pkt_rx;
1010                 in->obj = port->obj;
1011         }
1012
1013         return 0;
1014 }
1015
1016 static void
1017 port_in_build_free(struct rte_swx_pipeline *p)
1018 {
1019         free(p->in);
1020         p->in = NULL;
1021 }
1022
1023 static void
1024 port_in_free(struct rte_swx_pipeline *p)
1025 {
1026         port_in_build_free(p);
1027
1028         /* Input ports. */
1029         for ( ; ; ) {
1030                 struct port_in *port;
1031
1032                 port = TAILQ_FIRST(&p->ports_in);
1033                 if (!port)
1034                         break;
1035
1036                 TAILQ_REMOVE(&p->ports_in, port, node);
1037                 port->type->ops.free(port->obj);
1038                 free(port);
1039         }
1040
1041         /* Input port types. */
1042         for ( ; ; ) {
1043                 struct port_in_type *elem;
1044
1045                 elem = TAILQ_FIRST(&p->port_in_types);
1046                 if (!elem)
1047                         break;
1048
1049                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1050                 free(elem);
1051         }
1052 }
1053
1054 /*
1055  * Output port.
1056  */
1057 static struct port_out_type *
1058 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1059 {
1060         struct port_out_type *elem;
1061
1062         if (!name)
1063                 return NULL;
1064
1065         TAILQ_FOREACH(elem, &p->port_out_types, node)
1066                 if (!strcmp(elem->name, name))
1067                         return elem;
1068
1069         return NULL;
1070 }
1071
1072 int
1073 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1074                                         const char *name,
1075                                         struct rte_swx_port_out_ops *ops)
1076 {
1077         struct port_out_type *elem;
1078
1079         CHECK(p, EINVAL);
1080         CHECK_NAME(name, EINVAL);
1081         CHECK(ops, EINVAL);
1082         CHECK(ops->create, EINVAL);
1083         CHECK(ops->free, EINVAL);
1084         CHECK(ops->pkt_tx, EINVAL);
1085         CHECK(ops->stats_read, EINVAL);
1086
1087         CHECK(!port_out_type_find(p, name), EEXIST);
1088
1089         /* Node allocation. */
1090         elem = calloc(1, sizeof(struct port_out_type));
1091         CHECK(elem, ENOMEM);
1092
1093         /* Node initialization. */
1094         strcpy(elem->name, name);
1095         memcpy(&elem->ops, ops, sizeof(*ops));
1096
1097         /* Node add to tailq. */
1098         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1099
1100         return 0;
1101 }
1102
1103 static struct port_out *
1104 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1105 {
1106         struct port_out *port;
1107
1108         TAILQ_FOREACH(port, &p->ports_out, node)
1109                 if (port->id == port_id)
1110                         return port;
1111
1112         return NULL;
1113 }
1114
1115 int
1116 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1117                                  uint32_t port_id,
1118                                  const char *port_type_name,
1119                                  void *args)
1120 {
1121         struct port_out_type *type = NULL;
1122         struct port_out *port = NULL;
1123         void *obj = NULL;
1124
1125         CHECK(p, EINVAL);
1126
1127         CHECK(!port_out_find(p, port_id), EINVAL);
1128
1129         CHECK_NAME(port_type_name, EINVAL);
1130         type = port_out_type_find(p, port_type_name);
1131         CHECK(type, EINVAL);
1132
1133         obj = type->ops.create(args);
1134         CHECK(obj, ENODEV);
1135
1136         /* Node allocation. */
1137         port = calloc(1, sizeof(struct port_out));
1138         CHECK(port, ENOMEM);
1139
1140         /* Node initialization. */
1141         port->type = type;
1142         port->obj = obj;
1143         port->id = port_id;
1144
1145         /* Node add to tailq. */
1146         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1147         if (p->n_ports_out < port_id + 1)
1148                 p->n_ports_out = port_id + 1;
1149
1150         return 0;
1151 }
1152
1153 static int
1154 port_out_build(struct rte_swx_pipeline *p)
1155 {
1156         struct port_out *port;
1157         uint32_t i;
1158
1159         CHECK(p->n_ports_out, EINVAL);
1160
1161         for (i = 0; i < p->n_ports_out; i++)
1162                 CHECK(port_out_find(p, i), EINVAL);
1163
1164         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1165         CHECK(p->out, ENOMEM);
1166
1167         TAILQ_FOREACH(port, &p->ports_out, node) {
1168                 struct port_out_runtime *out = &p->out[port->id];
1169
1170                 out->pkt_tx = port->type->ops.pkt_tx;
1171                 out->flush = port->type->ops.flush;
1172                 out->obj = port->obj;
1173         }
1174
1175         return 0;
1176 }
1177
1178 static void
1179 port_out_build_free(struct rte_swx_pipeline *p)
1180 {
1181         free(p->out);
1182         p->out = NULL;
1183 }
1184
1185 static void
1186 port_out_free(struct rte_swx_pipeline *p)
1187 {
1188         port_out_build_free(p);
1189
1190         /* Output ports. */
1191         for ( ; ; ) {
1192                 struct port_out *port;
1193
1194                 port = TAILQ_FIRST(&p->ports_out);
1195                 if (!port)
1196                         break;
1197
1198                 TAILQ_REMOVE(&p->ports_out, port, node);
1199                 port->type->ops.free(port->obj);
1200                 free(port);
1201         }
1202
1203         /* Output port types. */
1204         for ( ; ; ) {
1205                 struct port_out_type *elem;
1206
1207                 elem = TAILQ_FIRST(&p->port_out_types);
1208                 if (!elem)
1209                         break;
1210
1211                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1212                 free(elem);
1213         }
1214 }
1215
1216 /*
1217  * Extern object.
1218  */
1219 static struct extern_type *
1220 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1221 {
1222         struct extern_type *elem;
1223
1224         TAILQ_FOREACH(elem, &p->extern_types, node)
1225                 if (strcmp(elem->name, name) == 0)
1226                         return elem;
1227
1228         return NULL;
1229 }
1230
1231 static struct extern_type_member_func *
1232 extern_type_member_func_find(struct extern_type *type, const char *name)
1233 {
1234         struct extern_type_member_func *elem;
1235
1236         TAILQ_FOREACH(elem, &type->funcs, node)
1237                 if (strcmp(elem->name, name) == 0)
1238                         return elem;
1239
1240         return NULL;
1241 }
1242
1243 static struct extern_obj *
1244 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1245 {
1246         struct extern_obj *elem;
1247
1248         TAILQ_FOREACH(elem, &p->extern_objs, node)
1249                 if (strcmp(elem->name, name) == 0)
1250                         return elem;
1251
1252         return NULL;
1253 }
1254
1255 static struct field *
1256 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1257                                const char *name,
1258                                struct extern_obj **object)
1259 {
1260         struct extern_obj *obj;
1261         struct field *f;
1262         char *obj_name, *field_name;
1263
1264         if ((name[0] != 'e') || (name[1] != '.'))
1265                 return NULL;
1266
1267         obj_name = strdup(&name[2]);
1268         if (!obj_name)
1269                 return NULL;
1270
1271         field_name = strchr(obj_name, '.');
1272         if (!field_name) {
1273                 free(obj_name);
1274                 return NULL;
1275         }
1276
1277         *field_name = 0;
1278         field_name++;
1279
1280         obj = extern_obj_find(p, obj_name);
1281         if (!obj) {
1282                 free(obj_name);
1283                 return NULL;
1284         }
1285
1286         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1287         if (!f) {
1288                 free(obj_name);
1289                 return NULL;
1290         }
1291
1292         if (object)
1293                 *object = obj;
1294
1295         free(obj_name);
1296         return f;
1297 }
1298
1299 int
1300 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1301         const char *name,
1302         const char *mailbox_struct_type_name,
1303         rte_swx_extern_type_constructor_t constructor,
1304         rte_swx_extern_type_destructor_t destructor)
1305 {
1306         struct extern_type *elem;
1307         struct struct_type *mailbox_struct_type;
1308
1309         CHECK(p, EINVAL);
1310
1311         CHECK_NAME(name, EINVAL);
1312         CHECK(!extern_type_find(p, name), EEXIST);
1313
1314         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1315         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1316         CHECK(mailbox_struct_type, EINVAL);
1317
1318         CHECK(constructor, EINVAL);
1319         CHECK(destructor, EINVAL);
1320
1321         /* Node allocation. */
1322         elem = calloc(1, sizeof(struct extern_type));
1323         CHECK(elem, ENOMEM);
1324
1325         /* Node initialization. */
1326         strcpy(elem->name, name);
1327         elem->mailbox_struct_type = mailbox_struct_type;
1328         elem->constructor = constructor;
1329         elem->destructor = destructor;
1330         TAILQ_INIT(&elem->funcs);
1331
1332         /* Node add to tailq. */
1333         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1334
1335         return 0;
1336 }
1337
1338 int
1339 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1340         const char *extern_type_name,
1341         const char *name,
1342         rte_swx_extern_type_member_func_t member_func)
1343 {
1344         struct extern_type *type;
1345         struct extern_type_member_func *type_member;
1346
1347         CHECK(p, EINVAL);
1348
1349         CHECK(extern_type_name, EINVAL);
1350         type = extern_type_find(p, extern_type_name);
1351         CHECK(type, EINVAL);
1352         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1353
1354         CHECK(name, EINVAL);
1355         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1356
1357         CHECK(member_func, EINVAL);
1358
1359         /* Node allocation. */
1360         type_member = calloc(1, sizeof(struct extern_type_member_func));
1361         CHECK(type_member, ENOMEM);
1362
1363         /* Node initialization. */
1364         strcpy(type_member->name, name);
1365         type_member->func = member_func;
1366         type_member->id = type->n_funcs;
1367
1368         /* Node add to tailq. */
1369         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1370         type->n_funcs++;
1371
1372         return 0;
1373 }
1374
1375 int
1376 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1377                                       const char *extern_type_name,
1378                                       const char *name,
1379                                       const char *args)
1380 {
1381         struct extern_type *type;
1382         struct extern_obj *obj;
1383         void *obj_handle;
1384
1385         CHECK(p, EINVAL);
1386
1387         CHECK_NAME(extern_type_name, EINVAL);
1388         type = extern_type_find(p, extern_type_name);
1389         CHECK(type, EINVAL);
1390
1391         CHECK_NAME(name, EINVAL);
1392         CHECK(!extern_obj_find(p, name), EEXIST);
1393
1394         /* Node allocation. */
1395         obj = calloc(1, sizeof(struct extern_obj));
1396         CHECK(obj, ENOMEM);
1397
1398         /* Object construction. */
1399         obj_handle = type->constructor(args);
1400         if (!obj_handle) {
1401                 free(obj);
1402                 CHECK(0, ENODEV);
1403         }
1404
1405         /* Node initialization. */
1406         strcpy(obj->name, name);
1407         obj->type = type;
1408         obj->obj = obj_handle;
1409         obj->struct_id = p->n_structs;
1410         obj->id = p->n_extern_objs;
1411
1412         /* Node add to tailq. */
1413         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1414         p->n_extern_objs++;
1415         p->n_structs++;
1416
1417         return 0;
1418 }
1419
1420 static int
1421 extern_obj_build(struct rte_swx_pipeline *p)
1422 {
1423         uint32_t i;
1424
1425         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1426                 struct thread *t = &p->threads[i];
1427                 struct extern_obj *obj;
1428
1429                 t->extern_objs = calloc(p->n_extern_objs,
1430                                         sizeof(struct extern_obj_runtime));
1431                 CHECK(t->extern_objs, ENOMEM);
1432
1433                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1434                         struct extern_obj_runtime *r =
1435                                 &t->extern_objs[obj->id];
1436                         struct extern_type_member_func *func;
1437                         uint32_t mailbox_size =
1438                                 obj->type->mailbox_struct_type->n_bits / 8;
1439
1440                         r->obj = obj->obj;
1441
1442                         r->mailbox = calloc(1, mailbox_size);
1443                         CHECK(r->mailbox, ENOMEM);
1444
1445                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1446                                 r->funcs[func->id] = func->func;
1447
1448                         t->structs[obj->struct_id] = r->mailbox;
1449                 }
1450         }
1451
1452         return 0;
1453 }
1454
1455 static void
1456 extern_obj_build_free(struct rte_swx_pipeline *p)
1457 {
1458         uint32_t i;
1459
1460         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1461                 struct thread *t = &p->threads[i];
1462                 uint32_t j;
1463
1464                 if (!t->extern_objs)
1465                         continue;
1466
1467                 for (j = 0; j < p->n_extern_objs; j++) {
1468                         struct extern_obj_runtime *r = &t->extern_objs[j];
1469
1470                         free(r->mailbox);
1471                 }
1472
1473                 free(t->extern_objs);
1474                 t->extern_objs = NULL;
1475         }
1476 }
1477
1478 static void
1479 extern_obj_free(struct rte_swx_pipeline *p)
1480 {
1481         extern_obj_build_free(p);
1482
1483         /* Extern objects. */
1484         for ( ; ; ) {
1485                 struct extern_obj *elem;
1486
1487                 elem = TAILQ_FIRST(&p->extern_objs);
1488                 if (!elem)
1489                         break;
1490
1491                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1492                 if (elem->obj)
1493                         elem->type->destructor(elem->obj);
1494                 free(elem);
1495         }
1496
1497         /* Extern types. */
1498         for ( ; ; ) {
1499                 struct extern_type *elem;
1500
1501                 elem = TAILQ_FIRST(&p->extern_types);
1502                 if (!elem)
1503                         break;
1504
1505                 TAILQ_REMOVE(&p->extern_types, elem, node);
1506
1507                 for ( ; ; ) {
1508                         struct extern_type_member_func *func;
1509
1510                         func = TAILQ_FIRST(&elem->funcs);
1511                         if (!func)
1512                                 break;
1513
1514                         TAILQ_REMOVE(&elem->funcs, func, node);
1515                         free(func);
1516                 }
1517
1518                 free(elem);
1519         }
1520 }
1521
1522 /*
1523  * Extern function.
1524  */
1525 static struct extern_func *
1526 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1527 {
1528         struct extern_func *elem;
1529
1530         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1531                 if (strcmp(elem->name, name) == 0)
1532                         return elem;
1533
1534         return NULL;
1535 }
1536
1537 static struct field *
1538 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1539                                 const char *name,
1540                                 struct extern_func **function)
1541 {
1542         struct extern_func *func;
1543         struct field *f;
1544         char *func_name, *field_name;
1545
1546         if ((name[0] != 'f') || (name[1] != '.'))
1547                 return NULL;
1548
1549         func_name = strdup(&name[2]);
1550         if (!func_name)
1551                 return NULL;
1552
1553         field_name = strchr(func_name, '.');
1554         if (!field_name) {
1555                 free(func_name);
1556                 return NULL;
1557         }
1558
1559         *field_name = 0;
1560         field_name++;
1561
1562         func = extern_func_find(p, func_name);
1563         if (!func) {
1564                 free(func_name);
1565                 return NULL;
1566         }
1567
1568         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1569         if (!f) {
1570                 free(func_name);
1571                 return NULL;
1572         }
1573
1574         if (function)
1575                 *function = func;
1576
1577         free(func_name);
1578         return f;
1579 }
1580
1581 int
1582 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1583                                       const char *name,
1584                                       const char *mailbox_struct_type_name,
1585                                       rte_swx_extern_func_t func)
1586 {
1587         struct extern_func *f;
1588         struct struct_type *mailbox_struct_type;
1589
1590         CHECK(p, EINVAL);
1591
1592         CHECK_NAME(name, EINVAL);
1593         CHECK(!extern_func_find(p, name), EEXIST);
1594
1595         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1596         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1597         CHECK(mailbox_struct_type, EINVAL);
1598
1599         CHECK(func, EINVAL);
1600
1601         /* Node allocation. */
1602         f = calloc(1, sizeof(struct extern_func));
1603         CHECK(func, ENOMEM);
1604
1605         /* Node initialization. */
1606         strcpy(f->name, name);
1607         f->mailbox_struct_type = mailbox_struct_type;
1608         f->func = func;
1609         f->struct_id = p->n_structs;
1610         f->id = p->n_extern_funcs;
1611
1612         /* Node add to tailq. */
1613         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1614         p->n_extern_funcs++;
1615         p->n_structs++;
1616
1617         return 0;
1618 }
1619
1620 static int
1621 extern_func_build(struct rte_swx_pipeline *p)
1622 {
1623         uint32_t i;
1624
1625         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1626                 struct thread *t = &p->threads[i];
1627                 struct extern_func *func;
1628
1629                 /* Memory allocation. */
1630                 t->extern_funcs = calloc(p->n_extern_funcs,
1631                                          sizeof(struct extern_func_runtime));
1632                 CHECK(t->extern_funcs, ENOMEM);
1633
1634                 /* Extern function. */
1635                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1636                         struct extern_func_runtime *r =
1637                                 &t->extern_funcs[func->id];
1638                         uint32_t mailbox_size =
1639                                 func->mailbox_struct_type->n_bits / 8;
1640
1641                         r->func = func->func;
1642
1643                         r->mailbox = calloc(1, mailbox_size);
1644                         CHECK(r->mailbox, ENOMEM);
1645
1646                         t->structs[func->struct_id] = r->mailbox;
1647                 }
1648         }
1649
1650         return 0;
1651 }
1652
1653 static void
1654 extern_func_build_free(struct rte_swx_pipeline *p)
1655 {
1656         uint32_t i;
1657
1658         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1659                 struct thread *t = &p->threads[i];
1660                 uint32_t j;
1661
1662                 if (!t->extern_funcs)
1663                         continue;
1664
1665                 for (j = 0; j < p->n_extern_funcs; j++) {
1666                         struct extern_func_runtime *r = &t->extern_funcs[j];
1667
1668                         free(r->mailbox);
1669                 }
1670
1671                 free(t->extern_funcs);
1672                 t->extern_funcs = NULL;
1673         }
1674 }
1675
1676 static void
1677 extern_func_free(struct rte_swx_pipeline *p)
1678 {
1679         extern_func_build_free(p);
1680
1681         for ( ; ; ) {
1682                 struct extern_func *elem;
1683
1684                 elem = TAILQ_FIRST(&p->extern_funcs);
1685                 if (!elem)
1686                         break;
1687
1688                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1689                 free(elem);
1690         }
1691 }
1692
1693 /*
1694  * Header.
1695  */
1696 static struct header *
1697 header_find(struct rte_swx_pipeline *p, const char *name)
1698 {
1699         struct header *elem;
1700
1701         TAILQ_FOREACH(elem, &p->headers, node)
1702                 if (strcmp(elem->name, name) == 0)
1703                         return elem;
1704
1705         return NULL;
1706 }
1707
1708 static struct header *
1709 header_parse(struct rte_swx_pipeline *p,
1710              const char *name)
1711 {
1712         if (name[0] != 'h' || name[1] != '.')
1713                 return NULL;
1714
1715         return header_find(p, &name[2]);
1716 }
1717
1718 static struct field *
1719 header_field_parse(struct rte_swx_pipeline *p,
1720                    const char *name,
1721                    struct header **header)
1722 {
1723         struct header *h;
1724         struct field *f;
1725         char *header_name, *field_name;
1726
1727         if ((name[0] != 'h') || (name[1] != '.'))
1728                 return NULL;
1729
1730         header_name = strdup(&name[2]);
1731         if (!header_name)
1732                 return NULL;
1733
1734         field_name = strchr(header_name, '.');
1735         if (!field_name) {
1736                 free(header_name);
1737                 return NULL;
1738         }
1739
1740         *field_name = 0;
1741         field_name++;
1742
1743         h = header_find(p, header_name);
1744         if (!h) {
1745                 free(header_name);
1746                 return NULL;
1747         }
1748
1749         f = struct_type_field_find(h->st, field_name);
1750         if (!f) {
1751                 free(header_name);
1752                 return NULL;
1753         }
1754
1755         if (header)
1756                 *header = h;
1757
1758         free(header_name);
1759         return f;
1760 }
1761
1762 int
1763 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1764                                         const char *name,
1765                                         const char *struct_type_name)
1766 {
1767         struct struct_type *st;
1768         struct header *h;
1769         size_t n_headers_max;
1770
1771         CHECK(p, EINVAL);
1772         CHECK_NAME(name, EINVAL);
1773         CHECK_NAME(struct_type_name, EINVAL);
1774
1775         CHECK(!header_find(p, name), EEXIST);
1776
1777         st = struct_type_find(p, struct_type_name);
1778         CHECK(st, EINVAL);
1779
1780         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1781         CHECK(p->n_headers < n_headers_max, ENOSPC);
1782
1783         /* Node allocation. */
1784         h = calloc(1, sizeof(struct header));
1785         CHECK(h, ENOMEM);
1786
1787         /* Node initialization. */
1788         strcpy(h->name, name);
1789         h->st = st;
1790         h->struct_id = p->n_structs;
1791         h->id = p->n_headers;
1792
1793         /* Node add to tailq. */
1794         TAILQ_INSERT_TAIL(&p->headers, h, node);
1795         p->n_headers++;
1796         p->n_structs++;
1797
1798         return 0;
1799 }
1800
1801 static int
1802 header_build(struct rte_swx_pipeline *p)
1803 {
1804         struct header *h;
1805         uint32_t n_bytes = 0, i;
1806
1807         TAILQ_FOREACH(h, &p->headers, node) {
1808                 n_bytes += h->st->n_bits / 8;
1809         }
1810
1811         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1812                 struct thread *t = &p->threads[i];
1813                 uint32_t offset = 0;
1814
1815                 t->headers = calloc(p->n_headers,
1816                                     sizeof(struct header_runtime));
1817                 CHECK(t->headers, ENOMEM);
1818
1819                 t->headers_out = calloc(p->n_headers,
1820                                         sizeof(struct header_out_runtime));
1821                 CHECK(t->headers_out, ENOMEM);
1822
1823                 t->header_storage = calloc(1, n_bytes);
1824                 CHECK(t->header_storage, ENOMEM);
1825
1826                 t->header_out_storage = calloc(1, n_bytes);
1827                 CHECK(t->header_out_storage, ENOMEM);
1828
1829                 TAILQ_FOREACH(h, &p->headers, node) {
1830                         uint8_t *header_storage;
1831
1832                         header_storage = &t->header_storage[offset];
1833                         offset += h->st->n_bits / 8;
1834
1835                         t->headers[h->id].ptr0 = header_storage;
1836                         t->structs[h->struct_id] = header_storage;
1837                 }
1838         }
1839
1840         return 0;
1841 }
1842
1843 static void
1844 header_build_free(struct rte_swx_pipeline *p)
1845 {
1846         uint32_t i;
1847
1848         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1849                 struct thread *t = &p->threads[i];
1850
1851                 free(t->headers_out);
1852                 t->headers_out = NULL;
1853
1854                 free(t->headers);
1855                 t->headers = NULL;
1856
1857                 free(t->header_out_storage);
1858                 t->header_out_storage = NULL;
1859
1860                 free(t->header_storage);
1861                 t->header_storage = NULL;
1862         }
1863 }
1864
1865 static void
1866 header_free(struct rte_swx_pipeline *p)
1867 {
1868         header_build_free(p);
1869
1870         for ( ; ; ) {
1871                 struct header *elem;
1872
1873                 elem = TAILQ_FIRST(&p->headers);
1874                 if (!elem)
1875                         break;
1876
1877                 TAILQ_REMOVE(&p->headers, elem, node);
1878                 free(elem);
1879         }
1880 }
1881
1882 /*
1883  * Meta-data.
1884  */
1885 static struct field *
1886 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1887 {
1888         if (!p->metadata_st)
1889                 return NULL;
1890
1891         if (name[0] != 'm' || name[1] != '.')
1892                 return NULL;
1893
1894         return struct_type_field_find(p->metadata_st, &name[2]);
1895 }
1896
1897 int
1898 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1899                                           const char *struct_type_name)
1900 {
1901         struct struct_type *st = NULL;
1902
1903         CHECK(p, EINVAL);
1904
1905         CHECK_NAME(struct_type_name, EINVAL);
1906         st  = struct_type_find(p, struct_type_name);
1907         CHECK(st, EINVAL);
1908         CHECK(!p->metadata_st, EINVAL);
1909
1910         p->metadata_st = st;
1911         p->metadata_struct_id = p->n_structs;
1912
1913         p->n_structs++;
1914
1915         return 0;
1916 }
1917
1918 static int
1919 metadata_build(struct rte_swx_pipeline *p)
1920 {
1921         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1922         uint32_t i;
1923
1924         /* Thread-level initialization. */
1925         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1926                 struct thread *t = &p->threads[i];
1927                 uint8_t *metadata;
1928
1929                 metadata = calloc(1, n_bytes);
1930                 CHECK(metadata, ENOMEM);
1931
1932                 t->metadata = metadata;
1933                 t->structs[p->metadata_struct_id] = metadata;
1934         }
1935
1936         return 0;
1937 }
1938
1939 static void
1940 metadata_build_free(struct rte_swx_pipeline *p)
1941 {
1942         uint32_t i;
1943
1944         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1945                 struct thread *t = &p->threads[i];
1946
1947                 free(t->metadata);
1948                 t->metadata = NULL;
1949         }
1950 }
1951
1952 static void
1953 metadata_free(struct rte_swx_pipeline *p)
1954 {
1955         metadata_build_free(p);
1956 }
1957
1958 /*
1959  * Instruction.
1960  */
1961 static struct field *
1962 action_field_parse(struct action *action, const char *name);
1963
1964 static struct field *
1965 struct_field_parse(struct rte_swx_pipeline *p,
1966                    struct action *action,
1967                    const char *name,
1968                    uint32_t *struct_id)
1969 {
1970         struct field *f;
1971
1972         switch (name[0]) {
1973         case 'h':
1974         {
1975                 struct header *header;
1976
1977                 f = header_field_parse(p, name, &header);
1978                 if (!f)
1979                         return NULL;
1980
1981                 *struct_id = header->struct_id;
1982                 return f;
1983         }
1984
1985         case 'm':
1986         {
1987                 f = metadata_field_parse(p, name);
1988                 if (!f)
1989                         return NULL;
1990
1991                 *struct_id = p->metadata_struct_id;
1992                 return f;
1993         }
1994
1995         case 't':
1996         {
1997                 if (!action)
1998                         return NULL;
1999
2000                 f = action_field_parse(action, name);
2001                 if (!f)
2002                         return NULL;
2003
2004                 *struct_id = 0;
2005                 return f;
2006         }
2007
2008         case 'e':
2009         {
2010                 struct extern_obj *obj;
2011
2012                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2013                 if (!f)
2014                         return NULL;
2015
2016                 *struct_id = obj->struct_id;
2017                 return f;
2018         }
2019
2020         case 'f':
2021         {
2022                 struct extern_func *func;
2023
2024                 f = extern_func_mailbox_field_parse(p, name, &func);
2025                 if (!f)
2026                         return NULL;
2027
2028                 *struct_id = func->struct_id;
2029                 return f;
2030         }
2031
2032         default:
2033                 return NULL;
2034         }
2035 }
2036
2037 static inline void
2038 pipeline_port_inc(struct rte_swx_pipeline *p)
2039 {
2040         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2041 }
2042
2043 static inline void
2044 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2045 {
2046         t->ip = p->instructions;
2047 }
2048
2049 static inline void
2050 thread_ip_inc(struct rte_swx_pipeline *p);
2051
2052 static inline void
2053 thread_ip_inc(struct rte_swx_pipeline *p)
2054 {
2055         struct thread *t = &p->threads[p->thread_id];
2056
2057         t->ip++;
2058 }
2059
2060 static inline void
2061 thread_ip_inc_cond(struct thread *t, int cond)
2062 {
2063         t->ip += cond;
2064 }
2065
2066 static inline void
2067 thread_yield(struct rte_swx_pipeline *p)
2068 {
2069         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2070 }
2071
2072 /*
2073  * rx.
2074  */
2075 static int
2076 instr_rx_translate(struct rte_swx_pipeline *p,
2077                    struct action *action,
2078                    char **tokens,
2079                    int n_tokens,
2080                    struct instruction *instr,
2081                    struct instruction_data *data __rte_unused)
2082 {
2083         struct field *f;
2084
2085         CHECK(!action, EINVAL);
2086         CHECK(n_tokens == 2, EINVAL);
2087
2088         f = metadata_field_parse(p, tokens[1]);
2089         CHECK(f, EINVAL);
2090
2091         instr->type = INSTR_RX;
2092         instr->io.io.offset = f->offset / 8;
2093         instr->io.io.n_bits = f->n_bits;
2094         return 0;
2095 }
2096
2097 static inline void
2098 instr_rx_exec(struct rte_swx_pipeline *p);
2099
2100 static inline void
2101 instr_rx_exec(struct rte_swx_pipeline *p)
2102 {
2103         struct thread *t = &p->threads[p->thread_id];
2104         struct instruction *ip = t->ip;
2105         struct port_in_runtime *port = &p->in[p->port_id];
2106         struct rte_swx_pkt *pkt = &t->pkt;
2107         int pkt_received;
2108
2109         /* Packet. */
2110         pkt_received = port->pkt_rx(port->obj, pkt);
2111         t->ptr = &pkt->pkt[pkt->offset];
2112         rte_prefetch0(t->ptr);
2113
2114         TRACE("[Thread %2u] rx %s from port %u\n",
2115               p->thread_id,
2116               pkt_received ? "1 pkt" : "0 pkts",
2117               p->port_id);
2118
2119         /* Headers. */
2120         t->valid_headers = 0;
2121         t->n_headers_out = 0;
2122
2123         /* Meta-data. */
2124         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2125
2126         /* Tables. */
2127         t->table_state = p->table_state;
2128
2129         /* Thread. */
2130         pipeline_port_inc(p);
2131         thread_ip_inc_cond(t, pkt_received);
2132         thread_yield(p);
2133 }
2134
2135 /*
2136  * tx.
2137  */
2138 static int
2139 instr_tx_translate(struct rte_swx_pipeline *p,
2140                    struct action *action __rte_unused,
2141                    char **tokens,
2142                    int n_tokens,
2143                    struct instruction *instr,
2144                    struct instruction_data *data __rte_unused)
2145 {
2146         struct field *f;
2147
2148         CHECK(n_tokens == 2, EINVAL);
2149
2150         f = metadata_field_parse(p, tokens[1]);
2151         CHECK(f, EINVAL);
2152
2153         instr->type = INSTR_TX;
2154         instr->io.io.offset = f->offset / 8;
2155         instr->io.io.n_bits = f->n_bits;
2156         return 0;
2157 }
2158
2159 static inline void
2160 emit_handler(struct thread *t)
2161 {
2162         struct header_out_runtime *h0 = &t->headers_out[0];
2163         struct header_out_runtime *h1 = &t->headers_out[1];
2164         uint32_t offset = 0, i;
2165
2166         /* No header change or header decapsulation. */
2167         if ((t->n_headers_out == 1) &&
2168             (h0->ptr + h0->n_bytes == t->ptr)) {
2169                 TRACE("Emit handler: no header change or header decap.\n");
2170
2171                 t->pkt.offset -= h0->n_bytes;
2172                 t->pkt.length += h0->n_bytes;
2173
2174                 return;
2175         }
2176
2177         /* Header encapsulation (optionally, with prior header decasulation). */
2178         if ((t->n_headers_out == 2) &&
2179             (h1->ptr + h1->n_bytes == t->ptr) &&
2180             (h0->ptr == h0->ptr0)) {
2181                 uint32_t offset;
2182
2183                 TRACE("Emit handler: header encapsulation.\n");
2184
2185                 offset = h0->n_bytes + h1->n_bytes;
2186                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2187                 t->pkt.offset -= offset;
2188                 t->pkt.length += offset;
2189
2190                 return;
2191         }
2192
2193         /* Header insertion. */
2194         /* TBD */
2195
2196         /* Header extraction. */
2197         /* TBD */
2198
2199         /* For any other case. */
2200         TRACE("Emit handler: complex case.\n");
2201
2202         for (i = 0; i < t->n_headers_out; i++) {
2203                 struct header_out_runtime *h = &t->headers_out[i];
2204
2205                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2206                 offset += h->n_bytes;
2207         }
2208
2209         if (offset) {
2210                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2211                 t->pkt.offset -= offset;
2212                 t->pkt.length += offset;
2213         }
2214 }
2215
2216 static inline void
2217 instr_tx_exec(struct rte_swx_pipeline *p);
2218
2219 static inline void
2220 instr_tx_exec(struct rte_swx_pipeline *p)
2221 {
2222         struct thread *t = &p->threads[p->thread_id];
2223         struct instruction *ip = t->ip;
2224         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2225         struct port_out_runtime *port = &p->out[port_id];
2226         struct rte_swx_pkt *pkt = &t->pkt;
2227
2228         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2229               p->thread_id,
2230               (uint32_t)port_id);
2231
2232         /* Headers. */
2233         emit_handler(t);
2234
2235         /* Packet. */
2236         port->pkt_tx(port->obj, pkt);
2237
2238         /* Thread. */
2239         thread_ip_reset(p, t);
2240         instr_rx_exec(p);
2241 }
2242
2243 /*
2244  * extract.
2245  */
2246 static int
2247 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2248                             struct action *action,
2249                             char **tokens,
2250                             int n_tokens,
2251                             struct instruction *instr,
2252                             struct instruction_data *data __rte_unused)
2253 {
2254         struct header *h;
2255
2256         CHECK(!action, EINVAL);
2257         CHECK(n_tokens == 2, EINVAL);
2258
2259         h = header_parse(p, tokens[1]);
2260         CHECK(h, EINVAL);
2261
2262         instr->type = INSTR_HDR_EXTRACT;
2263         instr->io.hdr.header_id[0] = h->id;
2264         instr->io.hdr.struct_id[0] = h->struct_id;
2265         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2266         return 0;
2267 }
2268
2269 static inline void
2270 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2271
2272 static inline void
2273 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2274 {
2275         struct thread *t = &p->threads[p->thread_id];
2276         struct instruction *ip = t->ip;
2277         uint64_t valid_headers = t->valid_headers;
2278         uint8_t *ptr = t->ptr;
2279         uint32_t offset = t->pkt.offset;
2280         uint32_t length = t->pkt.length;
2281         uint32_t i;
2282
2283         for (i = 0; i < n_extract; i++) {
2284                 uint32_t header_id = ip->io.hdr.header_id[i];
2285                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2286                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2287
2288                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2289                       p->thread_id,
2290                       header_id,
2291                       n_bytes);
2292
2293                 /* Headers. */
2294                 t->structs[struct_id] = ptr;
2295                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2296
2297                 /* Packet. */
2298                 offset += n_bytes;
2299                 length -= n_bytes;
2300                 ptr += n_bytes;
2301         }
2302
2303         /* Headers. */
2304         t->valid_headers = valid_headers;
2305
2306         /* Packet. */
2307         t->pkt.offset = offset;
2308         t->pkt.length = length;
2309         t->ptr = ptr;
2310 }
2311
2312 static inline void
2313 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2314 {
2315         __instr_hdr_extract_exec(p, 1);
2316
2317         /* Thread. */
2318         thread_ip_inc(p);
2319 }
2320
2321 static inline void
2322 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2323 {
2324         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2325               p->thread_id);
2326
2327         __instr_hdr_extract_exec(p, 2);
2328
2329         /* Thread. */
2330         thread_ip_inc(p);
2331 }
2332
2333 static inline void
2334 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2335 {
2336         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2337               p->thread_id);
2338
2339         __instr_hdr_extract_exec(p, 3);
2340
2341         /* Thread. */
2342         thread_ip_inc(p);
2343 }
2344
2345 static inline void
2346 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2347 {
2348         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2349               p->thread_id);
2350
2351         __instr_hdr_extract_exec(p, 4);
2352
2353         /* Thread. */
2354         thread_ip_inc(p);
2355 }
2356
2357 static inline void
2358 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2359 {
2360         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2361               p->thread_id);
2362
2363         __instr_hdr_extract_exec(p, 5);
2364
2365         /* Thread. */
2366         thread_ip_inc(p);
2367 }
2368
2369 static inline void
2370 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2371 {
2372         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2373               p->thread_id);
2374
2375         __instr_hdr_extract_exec(p, 6);
2376
2377         /* Thread. */
2378         thread_ip_inc(p);
2379 }
2380
2381 static inline void
2382 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2383 {
2384         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2385               p->thread_id);
2386
2387         __instr_hdr_extract_exec(p, 7);
2388
2389         /* Thread. */
2390         thread_ip_inc(p);
2391 }
2392
2393 static inline void
2394 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2395 {
2396         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2397               p->thread_id);
2398
2399         __instr_hdr_extract_exec(p, 8);
2400
2401         /* Thread. */
2402         thread_ip_inc(p);
2403 }
2404
2405 /*
2406  * emit.
2407  */
2408 static int
2409 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2410                          struct action *action __rte_unused,
2411                          char **tokens,
2412                          int n_tokens,
2413                          struct instruction *instr,
2414                          struct instruction_data *data __rte_unused)
2415 {
2416         struct header *h;
2417
2418         CHECK(n_tokens == 2, EINVAL);
2419
2420         h = header_parse(p, tokens[1]);
2421         CHECK(h, EINVAL);
2422
2423         instr->type = INSTR_HDR_EMIT;
2424         instr->io.hdr.header_id[0] = h->id;
2425         instr->io.hdr.struct_id[0] = h->struct_id;
2426         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2427         return 0;
2428 }
2429
2430 static inline void
2431 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
2432
2433 static inline void
2434 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
2435 {
2436         struct thread *t = &p->threads[p->thread_id];
2437         struct instruction *ip = t->ip;
2438         uint32_t n_headers_out = t->n_headers_out;
2439         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
2440         uint8_t *ho_ptr = NULL;
2441         uint32_t ho_nbytes = 0, i;
2442
2443         for (i = 0; i < n_emit; i++) {
2444                 uint32_t header_id = ip->io.hdr.header_id[i];
2445                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2446                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2447
2448                 struct header_runtime *hi = &t->headers[header_id];
2449                 uint8_t *hi_ptr = t->structs[struct_id];
2450
2451                 TRACE("[Thread %2u]: emit header %u\n",
2452                       p->thread_id,
2453                       header_id);
2454
2455                 /* Headers. */
2456                 if (!i) {
2457                         if (!t->n_headers_out) {
2458                                 ho = &t->headers_out[0];
2459
2460                                 ho->ptr0 = hi->ptr0;
2461                                 ho->ptr = hi_ptr;
2462
2463                                 ho_ptr = hi_ptr;
2464                                 ho_nbytes = n_bytes;
2465
2466                                 n_headers_out = 1;
2467
2468                                 continue;
2469                         } else {
2470                                 ho_ptr = ho->ptr;
2471                                 ho_nbytes = ho->n_bytes;
2472                         }
2473                 }
2474
2475                 if (ho_ptr + ho_nbytes == hi_ptr) {
2476                         ho_nbytes += n_bytes;
2477                 } else {
2478                         ho->n_bytes = ho_nbytes;
2479
2480                         ho++;
2481                         ho->ptr0 = hi->ptr0;
2482                         ho->ptr = hi_ptr;
2483
2484                         ho_ptr = hi_ptr;
2485                         ho_nbytes = n_bytes;
2486
2487                         n_headers_out++;
2488                 }
2489         }
2490
2491         ho->n_bytes = ho_nbytes;
2492         t->n_headers_out = n_headers_out;
2493 }
2494
2495 static inline void
2496 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2497 {
2498         __instr_hdr_emit_exec(p, 1);
2499
2500         /* Thread. */
2501         thread_ip_inc(p);
2502 }
2503
2504 static inline void
2505 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2506 {
2507         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2508               p->thread_id);
2509
2510         __instr_hdr_emit_exec(p, 1);
2511         instr_tx_exec(p);
2512 }
2513
2514 static inline void
2515 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2516 {
2517         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2518               p->thread_id);
2519
2520         __instr_hdr_emit_exec(p, 2);
2521         instr_tx_exec(p);
2522 }
2523
2524 static inline void
2525 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2526 {
2527         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2528               p->thread_id);
2529
2530         __instr_hdr_emit_exec(p, 3);
2531         instr_tx_exec(p);
2532 }
2533
2534 static inline void
2535 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2536 {
2537         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2538               p->thread_id);
2539
2540         __instr_hdr_emit_exec(p, 4);
2541         instr_tx_exec(p);
2542 }
2543
2544 static inline void
2545 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2546 {
2547         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2548               p->thread_id);
2549
2550         __instr_hdr_emit_exec(p, 5);
2551         instr_tx_exec(p);
2552 }
2553
2554 static inline void
2555 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2556 {
2557         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2558               p->thread_id);
2559
2560         __instr_hdr_emit_exec(p, 6);
2561         instr_tx_exec(p);
2562 }
2563
2564 static inline void
2565 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2566 {
2567         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2568               p->thread_id);
2569
2570         __instr_hdr_emit_exec(p, 7);
2571         instr_tx_exec(p);
2572 }
2573
2574 static inline void
2575 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2576 {
2577         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2578               p->thread_id);
2579
2580         __instr_hdr_emit_exec(p, 8);
2581         instr_tx_exec(p);
2582 }
2583
2584 /*
2585  * validate.
2586  */
2587 static int
2588 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2589                              struct action *action __rte_unused,
2590                              char **tokens,
2591                              int n_tokens,
2592                              struct instruction *instr,
2593                              struct instruction_data *data __rte_unused)
2594 {
2595         struct header *h;
2596
2597         CHECK(n_tokens == 2, EINVAL);
2598
2599         h = header_parse(p, tokens[1]);
2600         CHECK(h, EINVAL);
2601
2602         instr->type = INSTR_HDR_VALIDATE;
2603         instr->valid.header_id = h->id;
2604         return 0;
2605 }
2606
2607 static inline void
2608 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2609 {
2610         struct thread *t = &p->threads[p->thread_id];
2611         struct instruction *ip = t->ip;
2612         uint32_t header_id = ip->valid.header_id;
2613
2614         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2615
2616         /* Headers. */
2617         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2618
2619         /* Thread. */
2620         thread_ip_inc(p);
2621 }
2622
2623 /*
2624  * invalidate.
2625  */
2626 static int
2627 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2628                                struct action *action __rte_unused,
2629                                char **tokens,
2630                                int n_tokens,
2631                                struct instruction *instr,
2632                                struct instruction_data *data __rte_unused)
2633 {
2634         struct header *h;
2635
2636         CHECK(n_tokens == 2, EINVAL);
2637
2638         h = header_parse(p, tokens[1]);
2639         CHECK(h, EINVAL);
2640
2641         instr->type = INSTR_HDR_INVALIDATE;
2642         instr->valid.header_id = h->id;
2643         return 0;
2644 }
2645
2646 static inline void
2647 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2648 {
2649         struct thread *t = &p->threads[p->thread_id];
2650         struct instruction *ip = t->ip;
2651         uint32_t header_id = ip->valid.header_id;
2652
2653         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2654
2655         /* Headers. */
2656         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2657
2658         /* Thread. */
2659         thread_ip_inc(p);
2660 }
2661
2662 /*
2663  * mov.
2664  */
2665 static int
2666 instr_mov_translate(struct rte_swx_pipeline *p,
2667                     struct action *action,
2668                     char **tokens,
2669                     int n_tokens,
2670                     struct instruction *instr,
2671                     struct instruction_data *data __rte_unused)
2672 {
2673         char *dst = tokens[1], *src = tokens[2];
2674         struct field *fdst, *fsrc;
2675         uint32_t dst_struct_id, src_struct_id, src_val;
2676
2677         CHECK(n_tokens == 3, EINVAL);
2678
2679         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2680         CHECK(fdst, EINVAL);
2681
2682         /* MOV or MOV_S. */
2683         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2684         if (fsrc) {
2685                 instr->type = INSTR_MOV;
2686                 if ((dst[0] == 'h' && src[0] != 'h') ||
2687                     (dst[0] != 'h' && src[0] == 'h'))
2688                         instr->type = INSTR_MOV_S;
2689
2690                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2691                 instr->mov.dst.n_bits = fdst->n_bits;
2692                 instr->mov.dst.offset = fdst->offset / 8;
2693                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2694                 instr->mov.src.n_bits = fsrc->n_bits;
2695                 instr->mov.src.offset = fsrc->offset / 8;
2696                 return 0;
2697         }
2698
2699         /* MOV_I. */
2700         src_val = strtoul(src, &src, 0);
2701         CHECK(!src[0], EINVAL);
2702
2703         if (dst[0] == 'h')
2704                 src_val = htonl(src_val);
2705
2706         instr->type = INSTR_MOV_I;
2707         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2708         instr->mov.dst.n_bits = fdst->n_bits;
2709         instr->mov.dst.offset = fdst->offset / 8;
2710         instr->mov.src_val = (uint32_t)src_val;
2711         return 0;
2712 }
2713
2714 static inline void
2715 instr_mov_exec(struct rte_swx_pipeline *p)
2716 {
2717         struct thread *t = &p->threads[p->thread_id];
2718         struct instruction *ip = t->ip;
2719
2720         TRACE("[Thread %2u] mov\n",
2721               p->thread_id);
2722
2723         MOV(t, ip);
2724
2725         /* Thread. */
2726         thread_ip_inc(p);
2727 }
2728
2729 static inline void
2730 instr_mov_s_exec(struct rte_swx_pipeline *p)
2731 {
2732         struct thread *t = &p->threads[p->thread_id];
2733         struct instruction *ip = t->ip;
2734
2735         TRACE("[Thread %2u] mov (s)\n",
2736               p->thread_id);
2737
2738         MOV_S(t, ip);
2739
2740         /* Thread. */
2741         thread_ip_inc(p);
2742 }
2743
2744 static inline void
2745 instr_mov_i_exec(struct rte_swx_pipeline *p)
2746 {
2747         struct thread *t = &p->threads[p->thread_id];
2748         struct instruction *ip = t->ip;
2749
2750         TRACE("[Thread %2u] mov m.f %x\n",
2751               p->thread_id,
2752               ip->mov.src_val);
2753
2754         MOV_I(t, ip);
2755
2756         /* Thread. */
2757         thread_ip_inc(p);
2758 }
2759
2760 /*
2761  * dma.
2762  */
2763 static int
2764 instr_dma_translate(struct rte_swx_pipeline *p,
2765                     struct action *action,
2766                     char **tokens,
2767                     int n_tokens,
2768                     struct instruction *instr,
2769                     struct instruction_data *data __rte_unused)
2770 {
2771         char *dst = tokens[1];
2772         char *src = tokens[2];
2773         struct header *h;
2774         struct field *tf;
2775
2776         CHECK(action, EINVAL);
2777         CHECK(n_tokens == 3, EINVAL);
2778
2779         h = header_parse(p, dst);
2780         CHECK(h, EINVAL);
2781
2782         tf = action_field_parse(action, src);
2783         CHECK(tf, EINVAL);
2784
2785         instr->type = INSTR_DMA_HT;
2786         instr->dma.dst.header_id[0] = h->id;
2787         instr->dma.dst.struct_id[0] = h->struct_id;
2788         instr->dma.n_bytes[0] = h->st->n_bits / 8;
2789         instr->dma.src.offset[0] = tf->offset / 8;
2790
2791         return 0;
2792 }
2793
2794 static inline void
2795 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2796
2797 static inline void
2798 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2799 {
2800         struct thread *t = &p->threads[p->thread_id];
2801         struct instruction *ip = t->ip;
2802         uint8_t *action_data = t->structs[0];
2803         uint64_t valid_headers = t->valid_headers;
2804         uint32_t i;
2805
2806         for (i = 0; i < n_dma; i++) {
2807                 uint32_t header_id = ip->dma.dst.header_id[i];
2808                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2809                 uint32_t offset = ip->dma.src.offset[i];
2810                 uint32_t n_bytes = ip->dma.n_bytes[i];
2811
2812                 struct header_runtime *h = &t->headers[header_id];
2813                 uint8_t *h_ptr0 = h->ptr0;
2814                 uint8_t *h_ptr = t->structs[struct_id];
2815
2816                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2817                         h_ptr : h_ptr0;
2818                 void *src = &action_data[offset];
2819
2820                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2821
2822                 /* Headers. */
2823                 memcpy(dst, src, n_bytes);
2824                 t->structs[struct_id] = dst;
2825                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2826         }
2827
2828         t->valid_headers = valid_headers;
2829 }
2830
2831 static inline void
2832 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2833 {
2834         __instr_dma_ht_exec(p, 1);
2835
2836         /* Thread. */
2837         thread_ip_inc(p);
2838 }
2839
2840 static inline void
2841 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2842 {
2843         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2844               p->thread_id);
2845
2846         __instr_dma_ht_exec(p, 2);
2847
2848         /* Thread. */
2849         thread_ip_inc(p);
2850 }
2851
2852 static inline void
2853 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2854 {
2855         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2856               p->thread_id);
2857
2858         __instr_dma_ht_exec(p, 3);
2859
2860         /* Thread. */
2861         thread_ip_inc(p);
2862 }
2863
2864 static inline void
2865 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2866 {
2867         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2868               p->thread_id);
2869
2870         __instr_dma_ht_exec(p, 4);
2871
2872         /* Thread. */
2873         thread_ip_inc(p);
2874 }
2875
2876 static inline void
2877 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2878 {
2879         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2880               p->thread_id);
2881
2882         __instr_dma_ht_exec(p, 5);
2883
2884         /* Thread. */
2885         thread_ip_inc(p);
2886 }
2887
2888 static inline void
2889 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2890 {
2891         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2892               p->thread_id);
2893
2894         __instr_dma_ht_exec(p, 6);
2895
2896         /* Thread. */
2897         thread_ip_inc(p);
2898 }
2899
2900 static inline void
2901 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2902 {
2903         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2904               p->thread_id);
2905
2906         __instr_dma_ht_exec(p, 7);
2907
2908         /* Thread. */
2909         thread_ip_inc(p);
2910 }
2911
2912 static inline void
2913 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2914 {
2915         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2916               p->thread_id);
2917
2918         __instr_dma_ht_exec(p, 8);
2919
2920         /* Thread. */
2921         thread_ip_inc(p);
2922 }
2923
2924 /*
2925  * alu.
2926  */
2927 static int
2928 instr_alu_add_translate(struct rte_swx_pipeline *p,
2929                         struct action *action,
2930                         char **tokens,
2931                         int n_tokens,
2932                         struct instruction *instr,
2933                         struct instruction_data *data __rte_unused)
2934 {
2935         char *dst = tokens[1], *src = tokens[2];
2936         struct field *fdst, *fsrc;
2937         uint32_t dst_struct_id, src_struct_id, src_val;
2938
2939         CHECK(n_tokens == 3, EINVAL);
2940
2941         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2942         CHECK(fdst, EINVAL);
2943
2944         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2945         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2946         if (fsrc) {
2947                 instr->type = INSTR_ALU_ADD;
2948                 if (dst[0] == 'h' && src[0] == 'm')
2949                         instr->type = INSTR_ALU_ADD_HM;
2950                 if (dst[0] == 'm' && src[0] == 'h')
2951                         instr->type = INSTR_ALU_ADD_MH;
2952                 if (dst[0] == 'h' && src[0] == 'h')
2953                         instr->type = INSTR_ALU_ADD_HH;
2954
2955                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2956                 instr->alu.dst.n_bits = fdst->n_bits;
2957                 instr->alu.dst.offset = fdst->offset / 8;
2958                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2959                 instr->alu.src.n_bits = fsrc->n_bits;
2960                 instr->alu.src.offset = fsrc->offset / 8;
2961                 return 0;
2962         }
2963
2964         /* ADD_MI, ADD_HI. */
2965         src_val = strtoul(src, &src, 0);
2966         CHECK(!src[0], EINVAL);
2967
2968         instr->type = INSTR_ALU_ADD_MI;
2969         if (dst[0] == 'h')
2970                 instr->type = INSTR_ALU_ADD_HI;
2971
2972         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2973         instr->alu.dst.n_bits = fdst->n_bits;
2974         instr->alu.dst.offset = fdst->offset / 8;
2975         instr->alu.src_val = (uint32_t)src_val;
2976         return 0;
2977 }
2978
2979 static int
2980 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2981                         struct action *action,
2982                         char **tokens,
2983                         int n_tokens,
2984                         struct instruction *instr,
2985                         struct instruction_data *data __rte_unused)
2986 {
2987         char *dst = tokens[1], *src = tokens[2];
2988         struct field *fdst, *fsrc;
2989         uint32_t dst_struct_id, src_struct_id, src_val;
2990
2991         CHECK(n_tokens == 3, EINVAL);
2992
2993         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2994         CHECK(fdst, EINVAL);
2995
2996         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2997         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2998         if (fsrc) {
2999                 instr->type = INSTR_ALU_SUB;
3000                 if (dst[0] == 'h' && src[0] == 'm')
3001                         instr->type = INSTR_ALU_SUB_HM;
3002                 if (dst[0] == 'm' && src[0] == 'h')
3003                         instr->type = INSTR_ALU_SUB_MH;
3004                 if (dst[0] == 'h' && src[0] == 'h')
3005                         instr->type = INSTR_ALU_SUB_HH;
3006
3007                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3008                 instr->alu.dst.n_bits = fdst->n_bits;
3009                 instr->alu.dst.offset = fdst->offset / 8;
3010                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3011                 instr->alu.src.n_bits = fsrc->n_bits;
3012                 instr->alu.src.offset = fsrc->offset / 8;
3013                 return 0;
3014         }
3015
3016         /* SUB_MI, SUB_HI. */
3017         src_val = strtoul(src, &src, 0);
3018         CHECK(!src[0], EINVAL);
3019
3020         instr->type = INSTR_ALU_SUB_MI;
3021         if (dst[0] == 'h')
3022                 instr->type = INSTR_ALU_SUB_HI;
3023
3024         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3025         instr->alu.dst.n_bits = fdst->n_bits;
3026         instr->alu.dst.offset = fdst->offset / 8;
3027         instr->alu.src_val = (uint32_t)src_val;
3028         return 0;
3029 }
3030
3031 static int
3032 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3033                           struct action *action __rte_unused,
3034                           char **tokens,
3035                           int n_tokens,
3036                           struct instruction *instr,
3037                           struct instruction_data *data __rte_unused)
3038 {
3039         char *dst = tokens[1], *src = tokens[2];
3040         struct header *hdst, *hsrc;
3041         struct field *fdst, *fsrc;
3042
3043         CHECK(n_tokens == 3, EINVAL);
3044
3045         fdst = header_field_parse(p, dst, &hdst);
3046         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3047
3048         /* CKADD_FIELD. */
3049         fsrc = header_field_parse(p, src, &hsrc);
3050         if (fsrc) {
3051                 instr->type = INSTR_ALU_CKADD_FIELD;
3052                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3053                 instr->alu.dst.n_bits = fdst->n_bits;
3054                 instr->alu.dst.offset = fdst->offset / 8;
3055                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3056                 instr->alu.src.n_bits = fsrc->n_bits;
3057                 instr->alu.src.offset = fsrc->offset / 8;
3058                 return 0;
3059         }
3060
3061         /* CKADD_STRUCT, CKADD_STRUCT20. */
3062         hsrc = header_parse(p, src);
3063         CHECK(hsrc, EINVAL);
3064
3065         instr->type = INSTR_ALU_CKADD_STRUCT;
3066         if ((hsrc->st->n_bits / 8) == 20)
3067                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3068
3069         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3070         instr->alu.dst.n_bits = fdst->n_bits;
3071         instr->alu.dst.offset = fdst->offset / 8;
3072         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3073         instr->alu.src.n_bits = hsrc->st->n_bits;
3074         instr->alu.src.offset = 0; /* Unused. */
3075         return 0;
3076 }
3077
3078 static int
3079 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3080                           struct action *action __rte_unused,
3081                           char **tokens,
3082                           int n_tokens,
3083                           struct instruction *instr,
3084                           struct instruction_data *data __rte_unused)
3085 {
3086         char *dst = tokens[1], *src = tokens[2];
3087         struct header *hdst, *hsrc;
3088         struct field *fdst, *fsrc;
3089
3090         CHECK(n_tokens == 3, EINVAL);
3091
3092         fdst = header_field_parse(p, dst, &hdst);
3093         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3094
3095         fsrc = header_field_parse(p, src, &hsrc);
3096         CHECK(fsrc, EINVAL);
3097
3098         instr->type = INSTR_ALU_CKSUB_FIELD;
3099         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3100         instr->alu.dst.n_bits = fdst->n_bits;
3101         instr->alu.dst.offset = fdst->offset / 8;
3102         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3103         instr->alu.src.n_bits = fsrc->n_bits;
3104         instr->alu.src.offset = fsrc->offset / 8;
3105         return 0;
3106 }
3107
3108 static int
3109 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3110                         struct action *action,
3111                         char **tokens,
3112                         int n_tokens,
3113                         struct instruction *instr,
3114                         struct instruction_data *data __rte_unused)
3115 {
3116         char *dst = tokens[1], *src = tokens[2];
3117         struct field *fdst, *fsrc;
3118         uint32_t dst_struct_id, src_struct_id, src_val;
3119
3120         CHECK(n_tokens == 3, EINVAL);
3121
3122         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3123         CHECK(fdst, EINVAL);
3124
3125         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3126         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3127         if (fsrc) {
3128                 instr->type = INSTR_ALU_SHL;
3129                 if (dst[0] == 'h' && src[0] == 'm')
3130                         instr->type = INSTR_ALU_SHL_HM;
3131                 if (dst[0] == 'm' && src[0] == 'h')
3132                         instr->type = INSTR_ALU_SHL_MH;
3133                 if (dst[0] == 'h' && src[0] == 'h')
3134                         instr->type = INSTR_ALU_SHL_HH;
3135
3136                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3137                 instr->alu.dst.n_bits = fdst->n_bits;
3138                 instr->alu.dst.offset = fdst->offset / 8;
3139                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3140                 instr->alu.src.n_bits = fsrc->n_bits;
3141                 instr->alu.src.offset = fsrc->offset / 8;
3142                 return 0;
3143         }
3144
3145         /* SHL_MI, SHL_HI. */
3146         src_val = strtoul(src, &src, 0);
3147         CHECK(!src[0], EINVAL);
3148
3149         instr->type = INSTR_ALU_SHL_MI;
3150         if (dst[0] == 'h')
3151                 instr->type = INSTR_ALU_SHL_HI;
3152
3153         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3154         instr->alu.dst.n_bits = fdst->n_bits;
3155         instr->alu.dst.offset = fdst->offset / 8;
3156         instr->alu.src_val = (uint32_t)src_val;
3157         return 0;
3158 }
3159
3160 static int
3161 instr_alu_and_translate(struct rte_swx_pipeline *p,
3162                         struct action *action,
3163                         char **tokens,
3164                         int n_tokens,
3165                         struct instruction *instr,
3166                         struct instruction_data *data __rte_unused)
3167 {
3168         char *dst = tokens[1], *src = tokens[2];
3169         struct field *fdst, *fsrc;
3170         uint32_t dst_struct_id, src_struct_id, src_val;
3171
3172         CHECK(n_tokens == 3, EINVAL);
3173
3174         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3175         CHECK(fdst, EINVAL);
3176
3177         /* AND or AND_S. */
3178         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3179         if (fsrc) {
3180                 instr->type = INSTR_ALU_AND;
3181                 if ((dst[0] == 'h' && src[0] != 'h') ||
3182                     (dst[0] != 'h' && src[0] == 'h'))
3183                         instr->type = INSTR_ALU_AND_S;
3184
3185                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3186                 instr->alu.dst.n_bits = fdst->n_bits;
3187                 instr->alu.dst.offset = fdst->offset / 8;
3188                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3189                 instr->alu.src.n_bits = fsrc->n_bits;
3190                 instr->alu.src.offset = fsrc->offset / 8;
3191                 return 0;
3192         }
3193
3194         /* AND_I. */
3195         src_val = strtoul(src, &src, 0);
3196         CHECK(!src[0], EINVAL);
3197
3198         if (dst[0] == 'h')
3199                 src_val = htonl(src_val);
3200
3201         instr->type = INSTR_ALU_AND_I;
3202         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3203         instr->alu.dst.n_bits = fdst->n_bits;
3204         instr->alu.dst.offset = fdst->offset / 8;
3205         instr->alu.src_val = (uint32_t)src_val;
3206         return 0;
3207 }
3208
3209 static int
3210 instr_alu_or_translate(struct rte_swx_pipeline *p,
3211                        struct action *action,
3212                        char **tokens,
3213                        int n_tokens,
3214                        struct instruction *instr,
3215                        struct instruction_data *data __rte_unused)
3216 {
3217         char *dst = tokens[1], *src = tokens[2];
3218         struct field *fdst, *fsrc;
3219         uint32_t dst_struct_id, src_struct_id, src_val;
3220
3221         CHECK(n_tokens == 3, EINVAL);
3222
3223         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3224         CHECK(fdst, EINVAL);
3225
3226         /* OR or OR_S. */
3227         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3228         if (fsrc) {
3229                 instr->type = INSTR_ALU_OR;
3230                 if ((dst[0] == 'h' && src[0] != 'h') ||
3231                     (dst[0] != 'h' && src[0] == 'h'))
3232                         instr->type = INSTR_ALU_OR_S;
3233
3234                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3235                 instr->alu.dst.n_bits = fdst->n_bits;
3236                 instr->alu.dst.offset = fdst->offset / 8;
3237                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3238                 instr->alu.src.n_bits = fsrc->n_bits;
3239                 instr->alu.src.offset = fsrc->offset / 8;
3240                 return 0;
3241         }
3242
3243         /* OR_I. */
3244         src_val = strtoul(src, &src, 0);
3245         CHECK(!src[0], EINVAL);
3246
3247         if (dst[0] == 'h')
3248                 src_val = htonl(src_val);
3249
3250         instr->type = INSTR_ALU_OR_I;
3251         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3252         instr->alu.dst.n_bits = fdst->n_bits;
3253         instr->alu.dst.offset = fdst->offset / 8;
3254         instr->alu.src_val = (uint32_t)src_val;
3255         return 0;
3256 }
3257
3258 static int
3259 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3260                         struct action *action,
3261                         char **tokens,
3262                         int n_tokens,
3263                         struct instruction *instr,
3264                         struct instruction_data *data __rte_unused)
3265 {
3266         char *dst = tokens[1], *src = tokens[2];
3267         struct field *fdst, *fsrc;
3268         uint32_t dst_struct_id, src_struct_id, src_val;
3269
3270         CHECK(n_tokens == 3, EINVAL);
3271
3272         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3273         CHECK(fdst, EINVAL);
3274
3275         /* XOR or XOR_S. */
3276         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3277         if (fsrc) {
3278                 instr->type = INSTR_ALU_XOR;
3279                 if ((dst[0] == 'h' && src[0] != 'h') ||
3280                     (dst[0] != 'h' && src[0] == 'h'))
3281                         instr->type = INSTR_ALU_XOR_S;
3282
3283                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3284                 instr->alu.dst.n_bits = fdst->n_bits;
3285                 instr->alu.dst.offset = fdst->offset / 8;
3286                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3287                 instr->alu.src.n_bits = fsrc->n_bits;
3288                 instr->alu.src.offset = fsrc->offset / 8;
3289                 return 0;
3290         }
3291
3292         /* XOR_I. */
3293         src_val = strtoul(src, &src, 0);
3294         CHECK(!src[0], EINVAL);
3295
3296         if (dst[0] == 'h')
3297                 src_val = htonl(src_val);
3298
3299         instr->type = INSTR_ALU_XOR_I;
3300         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3301         instr->alu.dst.n_bits = fdst->n_bits;
3302         instr->alu.dst.offset = fdst->offset / 8;
3303         instr->alu.src_val = (uint32_t)src_val;
3304         return 0;
3305 }
3306
3307 static inline void
3308 instr_alu_add_exec(struct rte_swx_pipeline *p)
3309 {
3310         struct thread *t = &p->threads[p->thread_id];
3311         struct instruction *ip = t->ip;
3312
3313         TRACE("[Thread %2u] add\n", p->thread_id);
3314
3315         /* Structs. */
3316         ALU(t, ip, +);
3317
3318         /* Thread. */
3319         thread_ip_inc(p);
3320 }
3321
3322 static inline void
3323 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3324 {
3325         struct thread *t = &p->threads[p->thread_id];
3326         struct instruction *ip = t->ip;
3327
3328         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3329
3330         /* Structs. */
3331         ALU_MH(t, ip, +);
3332
3333         /* Thread. */
3334         thread_ip_inc(p);
3335 }
3336
3337 static inline void
3338 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3339 {
3340         struct thread *t = &p->threads[p->thread_id];
3341         struct instruction *ip = t->ip;
3342
3343         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3344
3345         /* Structs. */
3346         ALU_HM(t, ip, +);
3347
3348         /* Thread. */
3349         thread_ip_inc(p);
3350 }
3351
3352 static inline void
3353 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3354 {
3355         struct thread *t = &p->threads[p->thread_id];
3356         struct instruction *ip = t->ip;
3357
3358         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3359
3360         /* Structs. */
3361         ALU_HH(t, ip, +);
3362
3363         /* Thread. */
3364         thread_ip_inc(p);
3365 }
3366
3367 static inline void
3368 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3369 {
3370         struct thread *t = &p->threads[p->thread_id];
3371         struct instruction *ip = t->ip;
3372
3373         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3374
3375         /* Structs. */
3376         ALU_MI(t, ip, +);
3377
3378         /* Thread. */
3379         thread_ip_inc(p);
3380 }
3381
3382 static inline void
3383 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3384 {
3385         struct thread *t = &p->threads[p->thread_id];
3386         struct instruction *ip = t->ip;
3387
3388         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3389
3390         /* Structs. */
3391         ALU_HI(t, ip, +);
3392
3393         /* Thread. */
3394         thread_ip_inc(p);
3395 }
3396
3397 static inline void
3398 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3399 {
3400         struct thread *t = &p->threads[p->thread_id];
3401         struct instruction *ip = t->ip;
3402
3403         TRACE("[Thread %2u] sub\n", p->thread_id);
3404
3405         /* Structs. */
3406         ALU(t, ip, -);
3407
3408         /* Thread. */
3409         thread_ip_inc(p);
3410 }
3411
3412 static inline void
3413 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3414 {
3415         struct thread *t = &p->threads[p->thread_id];
3416         struct instruction *ip = t->ip;
3417
3418         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3419
3420         /* Structs. */
3421         ALU_MH(t, ip, -);
3422
3423         /* Thread. */
3424         thread_ip_inc(p);
3425 }
3426
3427 static inline void
3428 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3429 {
3430         struct thread *t = &p->threads[p->thread_id];
3431         struct instruction *ip = t->ip;
3432
3433         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3434
3435         /* Structs. */
3436         ALU_HM(t, ip, -);
3437
3438         /* Thread. */
3439         thread_ip_inc(p);
3440 }
3441
3442 static inline void
3443 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3444 {
3445         struct thread *t = &p->threads[p->thread_id];
3446         struct instruction *ip = t->ip;
3447
3448         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3449
3450         /* Structs. */
3451         ALU_HH(t, ip, -);
3452
3453         /* Thread. */
3454         thread_ip_inc(p);
3455 }
3456
3457 static inline void
3458 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3459 {
3460         struct thread *t = &p->threads[p->thread_id];
3461         struct instruction *ip = t->ip;
3462
3463         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3464
3465         /* Structs. */
3466         ALU_MI(t, ip, -);
3467
3468         /* Thread. */
3469         thread_ip_inc(p);
3470 }
3471
3472 static inline void
3473 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3474 {
3475         struct thread *t = &p->threads[p->thread_id];
3476         struct instruction *ip = t->ip;
3477
3478         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3479
3480         /* Structs. */
3481         ALU_HI(t, ip, -);
3482
3483         /* Thread. */
3484         thread_ip_inc(p);
3485 }
3486
3487 static inline void
3488 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3489 {
3490         struct thread *t = &p->threads[p->thread_id];
3491         struct instruction *ip = t->ip;
3492
3493         TRACE("[Thread %2u] shl\n", p->thread_id);
3494
3495         /* Structs. */
3496         ALU(t, ip, <<);
3497
3498         /* Thread. */
3499         thread_ip_inc(p);
3500 }
3501
3502 static inline void
3503 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3504 {
3505         struct thread *t = &p->threads[p->thread_id];
3506         struct instruction *ip = t->ip;
3507
3508         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
3509
3510         /* Structs. */
3511         ALU_MH(t, ip, <<);
3512
3513         /* Thread. */
3514         thread_ip_inc(p);
3515 }
3516
3517 static inline void
3518 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3519 {
3520         struct thread *t = &p->threads[p->thread_id];
3521         struct instruction *ip = t->ip;
3522
3523         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
3524
3525         /* Structs. */
3526         ALU_HM(t, ip, <<);
3527
3528         /* Thread. */
3529         thread_ip_inc(p);
3530 }
3531
3532 static inline void
3533 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3534 {
3535         struct thread *t = &p->threads[p->thread_id];
3536         struct instruction *ip = t->ip;
3537
3538         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
3539
3540         /* Structs. */
3541         ALU_HH(t, ip, <<);
3542
3543         /* Thread. */
3544         thread_ip_inc(p);
3545 }
3546
3547 static inline void
3548 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3549 {
3550         struct thread *t = &p->threads[p->thread_id];
3551         struct instruction *ip = t->ip;
3552
3553         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
3554
3555         /* Structs. */
3556         ALU_MI(t, ip, <<);
3557
3558         /* Thread. */
3559         thread_ip_inc(p);
3560 }
3561
3562 static inline void
3563 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3564 {
3565         struct thread *t = &p->threads[p->thread_id];
3566         struct instruction *ip = t->ip;
3567
3568         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
3569
3570         /* Structs. */
3571         ALU_HI(t, ip, <<);
3572
3573         /* Thread. */
3574         thread_ip_inc(p);
3575 }
3576
3577 static inline void
3578 instr_alu_and_exec(struct rte_swx_pipeline *p)
3579 {
3580         struct thread *t = &p->threads[p->thread_id];
3581         struct instruction *ip = t->ip;
3582
3583         TRACE("[Thread %2u] and\n", p->thread_id);
3584
3585         /* Structs. */
3586         ALU(t, ip, &);
3587
3588         /* Thread. */
3589         thread_ip_inc(p);
3590 }
3591
3592 static inline void
3593 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
3594 {
3595         struct thread *t = &p->threads[p->thread_id];
3596         struct instruction *ip = t->ip;
3597
3598         TRACE("[Thread %2u] and (s)\n", p->thread_id);
3599
3600         /* Structs. */
3601         ALU_S(t, ip, &);
3602
3603         /* Thread. */
3604         thread_ip_inc(p);
3605 }
3606
3607 static inline void
3608 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3609 {
3610         struct thread *t = &p->threads[p->thread_id];
3611         struct instruction *ip = t->ip;
3612
3613         TRACE("[Thread %2u] and (i)\n", p->thread_id);
3614
3615         /* Structs. */
3616         ALU_I(t, ip, &);
3617
3618         /* Thread. */
3619         thread_ip_inc(p);
3620 }
3621
3622 static inline void
3623 instr_alu_or_exec(struct rte_swx_pipeline *p)
3624 {
3625         struct thread *t = &p->threads[p->thread_id];
3626         struct instruction *ip = t->ip;
3627
3628         TRACE("[Thread %2u] or\n", p->thread_id);
3629
3630         /* Structs. */
3631         ALU(t, ip, |);
3632
3633         /* Thread. */
3634         thread_ip_inc(p);
3635 }
3636
3637 static inline void
3638 instr_alu_or_s_exec(struct rte_swx_pipeline *p)
3639 {
3640         struct thread *t = &p->threads[p->thread_id];
3641         struct instruction *ip = t->ip;
3642
3643         TRACE("[Thread %2u] or (s)\n", p->thread_id);
3644
3645         /* Structs. */
3646         ALU_S(t, ip, |);
3647
3648         /* Thread. */
3649         thread_ip_inc(p);
3650 }
3651
3652 static inline void
3653 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3654 {
3655         struct thread *t = &p->threads[p->thread_id];
3656         struct instruction *ip = t->ip;
3657
3658         TRACE("[Thread %2u] or (i)\n", p->thread_id);
3659
3660         /* Structs. */
3661         ALU_I(t, ip, |);
3662
3663         /* Thread. */
3664         thread_ip_inc(p);
3665 }
3666
3667 static inline void
3668 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3669 {
3670         struct thread *t = &p->threads[p->thread_id];
3671         struct instruction *ip = t->ip;
3672
3673         TRACE("[Thread %2u] xor\n", p->thread_id);
3674
3675         /* Structs. */
3676         ALU(t, ip, ^);
3677
3678         /* Thread. */
3679         thread_ip_inc(p);
3680 }
3681
3682 static inline void
3683 instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
3684 {
3685         struct thread *t = &p->threads[p->thread_id];
3686         struct instruction *ip = t->ip;
3687
3688         TRACE("[Thread %2u] xor (s)\n", p->thread_id);
3689
3690         /* Structs. */
3691         ALU_S(t, ip, ^);
3692
3693         /* Thread. */
3694         thread_ip_inc(p);
3695 }
3696
3697 static inline void
3698 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3699 {
3700         struct thread *t = &p->threads[p->thread_id];
3701         struct instruction *ip = t->ip;
3702
3703         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3704
3705         /* Structs. */
3706         ALU_I(t, ip, ^);
3707
3708         /* Thread. */
3709         thread_ip_inc(p);
3710 }
3711
3712 static inline void
3713 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3714 {
3715         struct thread *t = &p->threads[p->thread_id];
3716         struct instruction *ip = t->ip;
3717         uint8_t *dst_struct, *src_struct;
3718         uint16_t *dst16_ptr, dst;
3719         uint64_t *src64_ptr, src64, src64_mask, src;
3720         uint64_t r;
3721
3722         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3723
3724         /* Structs. */
3725         dst_struct = t->structs[ip->alu.dst.struct_id];
3726         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3727         dst = *dst16_ptr;
3728
3729         src_struct = t->structs[ip->alu.src.struct_id];
3730         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3731         src64 = *src64_ptr;
3732         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3733         src = src64 & src64_mask;
3734
3735         r = dst;
3736         r = ~r & 0xFFFF;
3737
3738         /* The first input (r) is a 16-bit number. The second and the third
3739          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3740          * three numbers (output r) is a 34-bit number.
3741          */
3742         r += (src >> 32) + (src & 0xFFFFFFFF);
3743
3744         /* The first input is a 16-bit number. The second input is an 18-bit
3745          * number. In the worst case scenario, the sum of the two numbers is a
3746          * 19-bit number.
3747          */
3748         r = (r & 0xFFFF) + (r >> 16);
3749
3750         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3751          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3752          */
3753         r = (r & 0xFFFF) + (r >> 16);
3754
3755         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3756          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3757          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3758          * therefore the output r is always a 16-bit number.
3759          */
3760         r = (r & 0xFFFF) + (r >> 16);
3761
3762         r = ~r & 0xFFFF;
3763         r = r ? r : 0xFFFF;
3764
3765         *dst16_ptr = (uint16_t)r;
3766
3767         /* Thread. */
3768         thread_ip_inc(p);
3769 }
3770
3771 static inline void
3772 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3773 {
3774         struct thread *t = &p->threads[p->thread_id];
3775         struct instruction *ip = t->ip;
3776         uint8_t *dst_struct, *src_struct;
3777         uint16_t *dst16_ptr, dst;
3778         uint64_t *src64_ptr, src64, src64_mask, src;
3779         uint64_t r;
3780
3781         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3782
3783         /* Structs. */
3784         dst_struct = t->structs[ip->alu.dst.struct_id];
3785         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3786         dst = *dst16_ptr;
3787
3788         src_struct = t->structs[ip->alu.src.struct_id];
3789         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3790         src64 = *src64_ptr;
3791         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3792         src = src64 & src64_mask;
3793
3794         r = dst;
3795         r = ~r & 0xFFFF;
3796
3797         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3798          * the following sequence of operations in 2's complement arithmetic:
3799          *    a '- b = (a - b) % 0xFFFF.
3800          *
3801          * In order to prevent an underflow for the below subtraction, in which
3802          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3803          * minuend), we first add a multiple of the 0xFFFF modulus to the
3804          * minuend. The number we add to the minuend needs to be a 34-bit number
3805          * or higher, so for readability reasons we picked the 36-bit multiple.
3806          * We are effectively turning the 16-bit minuend into a 36-bit number:
3807          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3808          */
3809         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3810
3811         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3812          * result (the output r) is a 36-bit number.
3813          */
3814         r -= (src >> 32) + (src & 0xFFFFFFFF);
3815
3816         /* The first input is a 16-bit number. The second input is a 20-bit
3817          * number. Their sum is a 21-bit number.
3818          */
3819         r = (r & 0xFFFF) + (r >> 16);
3820
3821         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3822          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3823          */
3824         r = (r & 0xFFFF) + (r >> 16);
3825
3826         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3827          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3828          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3829          * generated, therefore the output r is always a 16-bit number.
3830          */
3831         r = (r & 0xFFFF) + (r >> 16);
3832
3833         r = ~r & 0xFFFF;
3834         r = r ? r : 0xFFFF;
3835
3836         *dst16_ptr = (uint16_t)r;
3837
3838         /* Thread. */
3839         thread_ip_inc(p);
3840 }
3841
3842 static inline void
3843 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3844 {
3845         struct thread *t = &p->threads[p->thread_id];
3846         struct instruction *ip = t->ip;
3847         uint8_t *dst_struct, *src_struct;
3848         uint16_t *dst16_ptr;
3849         uint32_t *src32_ptr;
3850         uint64_t r0, r1;
3851
3852         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3853
3854         /* Structs. */
3855         dst_struct = t->structs[ip->alu.dst.struct_id];
3856         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3857
3858         src_struct = t->structs[ip->alu.src.struct_id];
3859         src32_ptr = (uint32_t *)&src_struct[0];
3860
3861         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
3862         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3863         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
3864         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3865         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3866
3867         /* The first input is a 16-bit number. The second input is a 19-bit
3868          * number. Their sum is a 20-bit number.
3869          */
3870         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3871
3872         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3873          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3874          */
3875         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3876
3877         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3878          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3879          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3880          * generated, therefore the output r is always a 16-bit number.
3881          */
3882         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3883
3884         r0 = ~r0 & 0xFFFF;
3885         r0 = r0 ? r0 : 0xFFFF;
3886
3887         *dst16_ptr = (uint16_t)r0;
3888
3889         /* Thread. */
3890         thread_ip_inc(p);
3891 }
3892
3893 static inline void
3894 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3895 {
3896         struct thread *t = &p->threads[p->thread_id];
3897         struct instruction *ip = t->ip;
3898         uint8_t *dst_struct, *src_struct;
3899         uint16_t *dst16_ptr;
3900         uint32_t *src32_ptr;
3901         uint64_t r = 0;
3902         uint32_t i;
3903
3904         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3905
3906         /* Structs. */
3907         dst_struct = t->structs[ip->alu.dst.struct_id];
3908         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3909
3910         src_struct = t->structs[ip->alu.src.struct_id];
3911         src32_ptr = (uint32_t *)&src_struct[0];
3912
3913         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
3914          * Therefore, in the worst case scenario, a 35-bit number is added to a
3915          * 16-bit number (the input r), so the output r is 36-bit number.
3916          */
3917         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
3918                 r += *src32_ptr;
3919
3920         /* The first input is a 16-bit number. The second input is a 20-bit
3921          * number. Their sum is a 21-bit number.
3922          */
3923         r = (r & 0xFFFF) + (r >> 16);
3924
3925         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3926          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
3927          */
3928         r = (r & 0xFFFF) + (r >> 16);
3929
3930         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3931          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3932          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3933          * generated, therefore the output r is always a 16-bit number.
3934          */
3935         r = (r & 0xFFFF) + (r >> 16);
3936
3937         r = ~r & 0xFFFF;
3938         r = r ? r : 0xFFFF;
3939
3940         *dst16_ptr = (uint16_t)r;
3941
3942         /* Thread. */
3943         thread_ip_inc(p);
3944 }
3945
3946 #define RTE_SWX_INSTRUCTION_TOKENS_MAX 16
3947
3948 static int
3949 instr_translate(struct rte_swx_pipeline *p,
3950                 struct action *action,
3951                 char *string,
3952                 struct instruction *instr,
3953                 struct instruction_data *data)
3954 {
3955         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
3956         int n_tokens = 0, tpos = 0;
3957
3958         /* Parse the instruction string into tokens. */
3959         for ( ; ; ) {
3960                 char *token;
3961
3962                 token = strtok_r(string, " \t\v", &string);
3963                 if (!token)
3964                         break;
3965
3966                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
3967
3968                 tokens[n_tokens] = token;
3969                 n_tokens++;
3970         }
3971
3972         CHECK(n_tokens, EINVAL);
3973
3974         /* Handle the optional instruction label. */
3975         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
3976                 strcpy(data->label, tokens[0]);
3977
3978                 tpos += 2;
3979                 CHECK(n_tokens - tpos, EINVAL);
3980         }
3981
3982         /* Identify the instruction type. */
3983         if (!strcmp(tokens[tpos], "rx"))
3984                 return instr_rx_translate(p,
3985                                           action,
3986                                           &tokens[tpos],
3987                                           n_tokens - tpos,
3988                                           instr,
3989                                           data);
3990
3991         if (!strcmp(tokens[tpos], "tx"))
3992                 return instr_tx_translate(p,
3993                                           action,
3994                                           &tokens[tpos],
3995                                           n_tokens - tpos,
3996                                           instr,
3997                                           data);
3998
3999         if (!strcmp(tokens[tpos], "extract"))
4000                 return instr_hdr_extract_translate(p,
4001                                                    action,
4002                                                    &tokens[tpos],
4003                                                    n_tokens - tpos,
4004                                                    instr,
4005                                                    data);
4006
4007         if (!strcmp(tokens[tpos], "emit"))
4008                 return instr_hdr_emit_translate(p,
4009                                                 action,
4010                                                 &tokens[tpos],
4011                                                 n_tokens - tpos,
4012                                                 instr,
4013                                                 data);
4014
4015         if (!strcmp(tokens[tpos], "validate"))
4016                 return instr_hdr_validate_translate(p,
4017                                                     action,
4018                                                     &tokens[tpos],
4019                                                     n_tokens - tpos,
4020                                                     instr,
4021                                                     data);
4022
4023         if (!strcmp(tokens[tpos], "invalidate"))
4024                 return instr_hdr_invalidate_translate(p,
4025                                                       action,
4026                                                       &tokens[tpos],
4027                                                       n_tokens - tpos,
4028                                                       instr,
4029                                                       data);
4030
4031         if (!strcmp(tokens[tpos], "mov"))
4032                 return instr_mov_translate(p,
4033                                            action,
4034                                            &tokens[tpos],
4035                                            n_tokens - tpos,
4036                                            instr,
4037                                            data);
4038
4039         if (!strcmp(tokens[tpos], "dma"))
4040                 return instr_dma_translate(p,
4041                                            action,
4042                                            &tokens[tpos],
4043                                            n_tokens - tpos,
4044                                            instr,
4045                                            data);
4046
4047         if (!strcmp(tokens[tpos], "add"))
4048                 return instr_alu_add_translate(p,
4049                                                action,
4050                                                &tokens[tpos],
4051                                                n_tokens - tpos,
4052                                                instr,
4053                                                data);
4054
4055         if (!strcmp(tokens[tpos], "sub"))
4056                 return instr_alu_sub_translate(p,
4057                                                action,
4058                                                &tokens[tpos],
4059                                                n_tokens - tpos,
4060                                                instr,
4061                                                data);
4062
4063         if (!strcmp(tokens[tpos], "ckadd"))
4064                 return instr_alu_ckadd_translate(p,
4065                                                  action,
4066                                                  &tokens[tpos],
4067                                                  n_tokens - tpos,
4068                                                  instr,
4069                                                  data);
4070
4071         if (!strcmp(tokens[tpos], "cksub"))
4072                 return instr_alu_cksub_translate(p,
4073                                                  action,
4074                                                  &tokens[tpos],
4075                                                  n_tokens - tpos,
4076                                                  instr,
4077                                                  data);
4078
4079         if (!strcmp(tokens[tpos], "and"))
4080                 return instr_alu_and_translate(p,
4081                                                action,
4082                                                &tokens[tpos],
4083                                                n_tokens - tpos,
4084                                                instr,
4085                                                data);
4086
4087         if (!strcmp(tokens[tpos], "or"))
4088                 return instr_alu_or_translate(p,
4089                                               action,
4090                                               &tokens[tpos],
4091                                               n_tokens - tpos,
4092                                               instr,
4093                                               data);
4094
4095         if (!strcmp(tokens[tpos], "xor"))
4096                 return instr_alu_xor_translate(p,
4097                                                action,
4098                                                &tokens[tpos],
4099                                                n_tokens - tpos,
4100                                                instr,
4101                                                data);
4102
4103         if (!strcmp(tokens[tpos], "shl"))
4104                 return instr_alu_shl_translate(p,
4105                                                action,
4106                                                &tokens[tpos],
4107                                                n_tokens - tpos,
4108                                                instr,
4109                                                data);
4110
4111         CHECK(0, EINVAL);
4112 }
4113
4114 static uint32_t
4115 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
4116 {
4117         uint32_t count = 0, i;
4118
4119         if (!label[0])
4120                 return 0;
4121
4122         for (i = 0; i < n; i++)
4123                 if (!strcmp(label, data[i].jmp_label))
4124                         count++;
4125
4126         return count;
4127 }
4128
4129 static int
4130 instr_label_check(struct instruction_data *instruction_data,
4131                   uint32_t n_instructions)
4132 {
4133         uint32_t i;
4134
4135         /* Check that all instruction labels are unique. */
4136         for (i = 0; i < n_instructions; i++) {
4137                 struct instruction_data *data = &instruction_data[i];
4138                 char *label = data->label;
4139                 uint32_t j;
4140
4141                 if (!label[0])
4142                         continue;
4143
4144                 for (j = i + 1; j < n_instructions; j++)
4145                         CHECK(strcmp(label, data[j].label), EINVAL);
4146         }
4147
4148         /* Get users for each instruction label. */
4149         for (i = 0; i < n_instructions; i++) {
4150                 struct instruction_data *data = &instruction_data[i];
4151                 char *label = data->label;
4152
4153                 data->n_users = label_is_used(instruction_data,
4154                                               n_instructions,
4155                                               label);
4156         }
4157
4158         return 0;
4159 }
4160
4161 static int
4162 instruction_config(struct rte_swx_pipeline *p,
4163                    struct action *a,
4164                    const char **instructions,
4165                    uint32_t n_instructions)
4166 {
4167         struct instruction *instr = NULL;
4168         struct instruction_data *data = NULL;
4169         char *string = NULL;
4170         int err = 0;
4171         uint32_t i;
4172
4173         CHECK(n_instructions, EINVAL);
4174         CHECK(instructions, EINVAL);
4175         for (i = 0; i < n_instructions; i++)
4176                 CHECK(instructions[i], EINVAL);
4177
4178         /* Memory allocation. */
4179         instr = calloc(n_instructions, sizeof(struct instruction));
4180         if (!instr) {
4181                 err = ENOMEM;
4182                 goto error;
4183         }
4184
4185         data = calloc(n_instructions, sizeof(struct instruction_data));
4186         if (!data) {
4187                 err = ENOMEM;
4188                 goto error;
4189         }
4190
4191         for (i = 0; i < n_instructions; i++) {
4192                 string = strdup(instructions[i]);
4193                 if (!string) {
4194                         err = ENOMEM;
4195                         goto error;
4196                 }
4197
4198                 err = instr_translate(p, a, string, &instr[i], &data[i]);
4199                 if (err)
4200                         goto error;
4201
4202                 free(string);
4203         }
4204
4205         err = instr_label_check(data, n_instructions);
4206         if (err)
4207                 goto error;
4208
4209         free(data);
4210
4211         if (a) {
4212                 a->instructions = instr;
4213                 a->n_instructions = n_instructions;
4214         } else {
4215                 p->instructions = instr;
4216                 p->n_instructions = n_instructions;
4217         }
4218
4219         return 0;
4220
4221 error:
4222         free(string);
4223         free(data);
4224         free(instr);
4225         return err;
4226 }
4227
4228 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
4229
4230 static instr_exec_t instruction_table[] = {
4231         [INSTR_RX] = instr_rx_exec,
4232         [INSTR_TX] = instr_tx_exec,
4233
4234         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
4235         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
4236         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
4237         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
4238         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
4239         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
4240         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
4241         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
4242
4243         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
4244         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
4245         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
4246         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
4247         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
4248         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
4249         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
4250         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
4251         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
4252
4253         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
4254         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
4255
4256         [INSTR_MOV] = instr_mov_exec,
4257         [INSTR_MOV_S] = instr_mov_s_exec,
4258         [INSTR_MOV_I] = instr_mov_i_exec,
4259
4260         [INSTR_DMA_HT] = instr_dma_ht_exec,
4261         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
4262         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
4263         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
4264         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
4265         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
4266         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
4267         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
4268
4269         [INSTR_ALU_ADD] = instr_alu_add_exec,
4270         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
4271         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
4272         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
4273         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
4274         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
4275
4276         [INSTR_ALU_SUB] = instr_alu_sub_exec,
4277         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
4278         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
4279         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
4280         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
4281         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
4282
4283         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
4284         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
4285         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
4286         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
4287
4288         [INSTR_ALU_AND] = instr_alu_and_exec,
4289         [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
4290         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
4291
4292         [INSTR_ALU_OR] = instr_alu_or_exec,
4293         [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
4294         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
4295
4296         [INSTR_ALU_XOR] = instr_alu_xor_exec,
4297         [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec,
4298         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
4299
4300         [INSTR_ALU_SHL] = instr_alu_shl_exec,
4301         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
4302         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
4303         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
4304         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
4305         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
4306 };
4307
4308 static inline void
4309 instr_exec(struct rte_swx_pipeline *p)
4310 {
4311         struct thread *t = &p->threads[p->thread_id];
4312         struct instruction *ip = t->ip;
4313         instr_exec_t instr = instruction_table[ip->type];
4314
4315         instr(p);
4316 }
4317
4318 /*
4319  * Action.
4320  */
4321 static struct action *
4322 action_find(struct rte_swx_pipeline *p, const char *name)
4323 {
4324         struct action *elem;
4325
4326         if (!name)
4327                 return NULL;
4328
4329         TAILQ_FOREACH(elem, &p->actions, node)
4330                 if (strcmp(elem->name, name) == 0)
4331                         return elem;
4332
4333         return NULL;
4334 }
4335
4336 static struct field *
4337 action_field_find(struct action *a, const char *name)
4338 {
4339         return a->st ? struct_type_field_find(a->st, name) : NULL;
4340 }
4341
4342 static struct field *
4343 action_field_parse(struct action *action, const char *name)
4344 {
4345         if (name[0] != 't' || name[1] != '.')
4346                 return NULL;
4347
4348         return action_field_find(action, &name[2]);
4349 }
4350
4351 int
4352 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
4353                                const char *name,
4354                                const char *args_struct_type_name,
4355                                const char **instructions,
4356                                uint32_t n_instructions)
4357 {
4358         struct struct_type *args_struct_type;
4359         struct action *a;
4360         int err;
4361
4362         CHECK(p, EINVAL);
4363
4364         CHECK_NAME(name, EINVAL);
4365         CHECK(!action_find(p, name), EEXIST);
4366
4367         if (args_struct_type_name) {
4368                 CHECK_NAME(args_struct_type_name, EINVAL);
4369                 args_struct_type = struct_type_find(p, args_struct_type_name);
4370                 CHECK(args_struct_type, EINVAL);
4371         } else {
4372                 args_struct_type = NULL;
4373         }
4374
4375         /* Node allocation. */
4376         a = calloc(1, sizeof(struct action));
4377         CHECK(a, ENOMEM);
4378
4379         /* Node initialization. */
4380         strcpy(a->name, name);
4381         a->st = args_struct_type;
4382         a->id = p->n_actions;
4383
4384         /* Instruction translation. */
4385         err = instruction_config(p, a, instructions, n_instructions);
4386         if (err) {
4387                 free(a);
4388                 return err;
4389         }
4390
4391         /* Node add to tailq. */
4392         TAILQ_INSERT_TAIL(&p->actions, a, node);
4393         p->n_actions++;
4394
4395         return 0;
4396 }
4397
4398 static int
4399 action_build(struct rte_swx_pipeline *p)
4400 {
4401         struct action *action;
4402
4403         p->action_instructions = calloc(p->n_actions,
4404                                         sizeof(struct instruction *));
4405         CHECK(p->action_instructions, ENOMEM);
4406
4407         TAILQ_FOREACH(action, &p->actions, node)
4408                 p->action_instructions[action->id] = action->instructions;
4409
4410         return 0;
4411 }
4412
4413 static void
4414 action_build_free(struct rte_swx_pipeline *p)
4415 {
4416         free(p->action_instructions);
4417         p->action_instructions = NULL;
4418 }
4419
4420 static void
4421 action_free(struct rte_swx_pipeline *p)
4422 {
4423         action_build_free(p);
4424
4425         for ( ; ; ) {
4426                 struct action *action;
4427
4428                 action = TAILQ_FIRST(&p->actions);
4429                 if (!action)
4430                         break;
4431
4432                 TAILQ_REMOVE(&p->actions, action, node);
4433                 free(action->instructions);
4434                 free(action);
4435         }
4436 }
4437
4438 /*
4439  * Table.
4440  */
4441 static struct table_type *
4442 table_type_find(struct rte_swx_pipeline *p, const char *name)
4443 {
4444         struct table_type *elem;
4445
4446         TAILQ_FOREACH(elem, &p->table_types, node)
4447                 if (strcmp(elem->name, name) == 0)
4448                         return elem;
4449
4450         return NULL;
4451 }
4452
4453 static struct table_type *
4454 table_type_resolve(struct rte_swx_pipeline *p,
4455                    const char *recommended_type_name,
4456                    enum rte_swx_table_match_type match_type)
4457 {
4458         struct table_type *elem;
4459
4460         /* Only consider the recommended type if the match type is correct. */
4461         if (recommended_type_name)
4462                 TAILQ_FOREACH(elem, &p->table_types, node)
4463                         if (!strcmp(elem->name, recommended_type_name) &&
4464                             (elem->match_type == match_type))
4465                                 return elem;
4466
4467         /* Ignore the recommended type and get the first element with this match
4468          * type.
4469          */
4470         TAILQ_FOREACH(elem, &p->table_types, node)
4471                 if (elem->match_type == match_type)
4472                         return elem;
4473
4474         return NULL;
4475 }
4476
4477 static struct table *
4478 table_find(struct rte_swx_pipeline *p, const char *name)
4479 {
4480         struct table *elem;
4481
4482         TAILQ_FOREACH(elem, &p->tables, node)
4483                 if (strcmp(elem->name, name) == 0)
4484                         return elem;
4485
4486         return NULL;
4487 }
4488
4489 static struct table *
4490 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
4491 {
4492         struct table *table = NULL;
4493
4494         TAILQ_FOREACH(table, &p->tables, node)
4495                 if (table->id == id)
4496                         return table;
4497
4498         return NULL;
4499 }
4500
4501 int
4502 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
4503                                      const char *name,
4504                                      enum rte_swx_table_match_type match_type,
4505                                      struct rte_swx_table_ops *ops)
4506 {
4507         struct table_type *elem;
4508
4509         CHECK(p, EINVAL);
4510
4511         CHECK_NAME(name, EINVAL);
4512         CHECK(!table_type_find(p, name), EEXIST);
4513
4514         CHECK(ops, EINVAL);
4515         CHECK(ops->create, EINVAL);
4516         CHECK(ops->lkp, EINVAL);
4517         CHECK(ops->free, EINVAL);
4518
4519         /* Node allocation. */
4520         elem = calloc(1, sizeof(struct table_type));
4521         CHECK(elem, ENOMEM);
4522
4523         /* Node initialization. */
4524         strcpy(elem->name, name);
4525         elem->match_type = match_type;
4526         memcpy(&elem->ops, ops, sizeof(*ops));
4527
4528         /* Node add to tailq. */
4529         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
4530
4531         return 0;
4532 }
4533
4534 static enum rte_swx_table_match_type
4535 table_match_type_resolve(struct rte_swx_match_field_params *fields,
4536                          uint32_t n_fields)
4537 {
4538         uint32_t i;
4539
4540         for (i = 0; i < n_fields; i++)
4541                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
4542                         break;
4543
4544         if (i == n_fields)
4545                 return RTE_SWX_TABLE_MATCH_EXACT;
4546
4547         if ((i == n_fields - 1) &&
4548             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
4549                 return RTE_SWX_TABLE_MATCH_LPM;
4550
4551         return RTE_SWX_TABLE_MATCH_WILDCARD;
4552 }
4553
4554 int
4555 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
4556                               const char *name,
4557                               struct rte_swx_pipeline_table_params *params,
4558                               const char *recommended_table_type_name,
4559                               const char *args,
4560                               uint32_t size)
4561 {
4562         struct table_type *type;
4563         struct table *t;
4564         struct action *default_action;
4565         struct header *header = NULL;
4566         int is_header = 0;
4567         uint32_t offset_prev = 0, action_data_size_max = 0, i;
4568
4569         CHECK(p, EINVAL);
4570
4571         CHECK_NAME(name, EINVAL);
4572         CHECK(!table_find(p, name), EEXIST);
4573
4574         CHECK(params, EINVAL);
4575
4576         /* Match checks. */
4577         CHECK(!params->n_fields || params->fields, EINVAL);
4578         for (i = 0; i < params->n_fields; i++) {
4579                 struct rte_swx_match_field_params *field = &params->fields[i];
4580                 struct header *h;
4581                 struct field *hf, *mf;
4582                 uint32_t offset;
4583
4584                 CHECK_NAME(field->name, EINVAL);
4585
4586                 hf = header_field_parse(p, field->name, &h);
4587                 mf = metadata_field_parse(p, field->name);
4588                 CHECK(hf || mf, EINVAL);
4589
4590                 offset = hf ? hf->offset : mf->offset;
4591
4592                 if (i == 0) {
4593                         is_header = hf ? 1 : 0;
4594                         header = hf ? h : NULL;
4595                         offset_prev = offset;
4596
4597                         continue;
4598                 }
4599
4600                 CHECK((is_header && hf && (h->id == header->id)) ||
4601                       (!is_header && mf), EINVAL);
4602
4603                 CHECK(offset > offset_prev, EINVAL);
4604                 offset_prev = offset;
4605         }
4606
4607         /* Action checks. */
4608         CHECK(params->n_actions, EINVAL);
4609         CHECK(params->action_names, EINVAL);
4610         for (i = 0; i < params->n_actions; i++) {
4611                 const char *action_name = params->action_names[i];
4612                 struct action *a;
4613                 uint32_t action_data_size;
4614
4615                 CHECK(action_name, EINVAL);
4616
4617                 a = action_find(p, action_name);
4618                 CHECK(a, EINVAL);
4619
4620                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
4621                 if (action_data_size > action_data_size_max)
4622                         action_data_size_max = action_data_size;
4623         }
4624
4625         CHECK(params->default_action_name, EINVAL);
4626         for (i = 0; i < p->n_actions; i++)
4627                 if (!strcmp(params->action_names[i],
4628                             params->default_action_name))
4629                         break;
4630         CHECK(i < params->n_actions, EINVAL);
4631         default_action = action_find(p, params->default_action_name);
4632         CHECK((default_action->st && params->default_action_data) ||
4633               !params->default_action_data, EINVAL);
4634
4635         /* Table type checks. */
4636         if (params->n_fields) {
4637                 enum rte_swx_table_match_type match_type;
4638
4639                 match_type = table_match_type_resolve(params->fields,
4640                                                       params->n_fields);
4641                 type = table_type_resolve(p,
4642                                           recommended_table_type_name,
4643                                           match_type);
4644                 CHECK(type, EINVAL);
4645         } else {
4646                 type = NULL;
4647         }
4648
4649         /* Memory allocation. */
4650         t = calloc(1, sizeof(struct table));
4651         CHECK(t, ENOMEM);
4652
4653         t->fields = calloc(params->n_fields, sizeof(struct match_field));
4654         if (!t->fields) {
4655                 free(t);
4656                 CHECK(0, ENOMEM);
4657         }
4658
4659         t->actions = calloc(params->n_actions, sizeof(struct action *));
4660         if (!t->actions) {
4661                 free(t->fields);
4662                 free(t);
4663                 CHECK(0, ENOMEM);
4664         }
4665
4666         if (action_data_size_max) {
4667                 t->default_action_data = calloc(1, action_data_size_max);
4668                 if (!t->default_action_data) {
4669                         free(t->actions);
4670                         free(t->fields);
4671                         free(t);
4672                         CHECK(0, ENOMEM);
4673                 }
4674         }
4675
4676         /* Node initialization. */
4677         strcpy(t->name, name);
4678         if (args && args[0])
4679                 strcpy(t->args, args);
4680         t->type = type;
4681
4682         for (i = 0; i < params->n_fields; i++) {
4683                 struct rte_swx_match_field_params *field = &params->fields[i];
4684                 struct match_field *f = &t->fields[i];
4685
4686                 f->match_type = field->match_type;
4687                 f->field = is_header ?
4688                         header_field_parse(p, field->name, NULL) :
4689                         metadata_field_parse(p, field->name);
4690         }
4691         t->n_fields = params->n_fields;
4692         t->is_header = is_header;
4693         t->header = header;
4694
4695         for (i = 0; i < params->n_actions; i++)
4696                 t->actions[i] = action_find(p, params->action_names[i]);
4697         t->default_action = default_action;
4698         if (default_action->st)
4699                 memcpy(t->default_action_data,
4700                        params->default_action_data,
4701                        default_action->st->n_bits / 8);
4702         t->n_actions = params->n_actions;
4703         t->default_action_is_const = params->default_action_is_const;
4704         t->action_data_size_max = action_data_size_max;
4705
4706         t->size = size;
4707         t->id = p->n_tables;
4708
4709         /* Node add to tailq. */
4710         TAILQ_INSERT_TAIL(&p->tables, t, node);
4711         p->n_tables++;
4712
4713         return 0;
4714 }
4715
4716 static struct rte_swx_table_params *
4717 table_params_get(struct table *table)
4718 {
4719         struct rte_swx_table_params *params;
4720         struct field *first, *last;
4721         uint8_t *key_mask;
4722         uint32_t key_size, key_offset, action_data_size, i;
4723
4724         /* Memory allocation. */
4725         params = calloc(1, sizeof(struct rte_swx_table_params));
4726         if (!params)
4727                 return NULL;
4728
4729         /* Key offset and size. */
4730         first = table->fields[0].field;
4731         last = table->fields[table->n_fields - 1].field;
4732         key_offset = first->offset / 8;
4733         key_size = (last->offset + last->n_bits - first->offset) / 8;
4734
4735         /* Memory allocation. */
4736         key_mask = calloc(1, key_size);
4737         if (!key_mask) {
4738                 free(params);
4739                 return NULL;
4740         }
4741
4742         /* Key mask. */
4743         for (i = 0; i < table->n_fields; i++) {
4744                 struct field *f = table->fields[i].field;
4745                 uint32_t start = (f->offset - first->offset) / 8;
4746                 size_t size = f->n_bits / 8;
4747
4748                 memset(&key_mask[start], 0xFF, size);
4749         }
4750
4751         /* Action data size. */
4752         action_data_size = 0;
4753         for (i = 0; i < table->n_actions; i++) {
4754                 struct action *action = table->actions[i];
4755                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
4756
4757                 if (ads > action_data_size)
4758                         action_data_size = ads;
4759         }
4760
4761         /* Fill in. */
4762         params->match_type = table->type->match_type;
4763         params->key_size = key_size;
4764         params->key_offset = key_offset;
4765         params->key_mask0 = key_mask;
4766         params->action_data_size = action_data_size;
4767         params->n_keys_max = table->size;
4768
4769         return params;
4770 }
4771
4772 static void
4773 table_params_free(struct rte_swx_table_params *params)
4774 {
4775         if (!params)
4776                 return;
4777
4778         free(params->key_mask0);
4779         free(params);
4780 }
4781
4782 static int
4783 table_state_build(struct rte_swx_pipeline *p)
4784 {
4785         struct table *table;
4786
4787         p->table_state = calloc(p->n_tables,
4788                                 sizeof(struct rte_swx_table_state));
4789         CHECK(p->table_state, ENOMEM);
4790
4791         TAILQ_FOREACH(table, &p->tables, node) {
4792                 struct rte_swx_table_state *ts = &p->table_state[table->id];
4793
4794                 if (table->type) {
4795                         struct rte_swx_table_params *params;
4796
4797                         /* ts->obj. */
4798                         params = table_params_get(table);
4799                         CHECK(params, ENOMEM);
4800
4801                         ts->obj = table->type->ops.create(params,
4802                                 NULL,
4803                                 table->args,
4804                                 p->numa_node);
4805
4806                         table_params_free(params);
4807                         CHECK(ts->obj, ENODEV);
4808                 }
4809
4810                 /* ts->default_action_data. */
4811                 if (table->action_data_size_max) {
4812                         ts->default_action_data =
4813                                 malloc(table->action_data_size_max);
4814                         CHECK(ts->default_action_data, ENOMEM);
4815
4816                         memcpy(ts->default_action_data,
4817                                table->default_action_data,
4818                                table->action_data_size_max);
4819                 }
4820
4821                 /* ts->default_action_id. */
4822                 ts->default_action_id = table->default_action->id;
4823         }
4824
4825         return 0;
4826 }
4827
4828 static void
4829 table_state_build_free(struct rte_swx_pipeline *p)
4830 {
4831         uint32_t i;
4832
4833         if (!p->table_state)
4834                 return;
4835
4836         for (i = 0; i < p->n_tables; i++) {
4837                 struct rte_swx_table_state *ts = &p->table_state[i];
4838                 struct table *table = table_find_by_id(p, i);
4839
4840                 /* ts->obj. */
4841                 if (table->type && ts->obj)
4842                         table->type->ops.free(ts->obj);
4843
4844                 /* ts->default_action_data. */
4845                 free(ts->default_action_data);
4846         }
4847
4848         free(p->table_state);
4849         p->table_state = NULL;
4850 }
4851
4852 static void
4853 table_state_free(struct rte_swx_pipeline *p)
4854 {
4855         table_state_build_free(p);
4856 }
4857
4858 static int
4859 table_stub_lkp(void *table __rte_unused,
4860                void *mailbox __rte_unused,
4861                uint8_t **key __rte_unused,
4862                uint64_t *action_id __rte_unused,
4863                uint8_t **action_data __rte_unused,
4864                int *hit)
4865 {
4866         *hit = 0;
4867         return 1; /* DONE. */
4868 }
4869
4870 static int
4871 table_build(struct rte_swx_pipeline *p)
4872 {
4873         uint32_t i;
4874
4875         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
4876                 struct thread *t = &p->threads[i];
4877                 struct table *table;
4878
4879                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
4880                 CHECK(t->tables, ENOMEM);
4881
4882                 TAILQ_FOREACH(table, &p->tables, node) {
4883                         struct table_runtime *r = &t->tables[table->id];
4884
4885                         if (table->type) {
4886                                 uint64_t size;
4887
4888                                 size = table->type->ops.mailbox_size_get();
4889
4890                                 /* r->func. */
4891                                 r->func = table->type->ops.lkp;
4892
4893                                 /* r->mailbox. */
4894                                 if (size) {
4895                                         r->mailbox = calloc(1, size);
4896                                         CHECK(r->mailbox, ENOMEM);
4897                                 }
4898
4899                                 /* r->key. */
4900                                 r->key = table->is_header ?
4901                                         &t->structs[table->header->struct_id] :
4902                                         &t->structs[p->metadata_struct_id];
4903                         } else {
4904                                 r->func = table_stub_lkp;
4905                         }
4906                 }
4907         }
4908
4909         return 0;
4910 }
4911
4912 static void
4913 table_build_free(struct rte_swx_pipeline *p)
4914 {
4915         uint32_t i;
4916
4917         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
4918                 struct thread *t = &p->threads[i];
4919                 uint32_t j;
4920
4921                 if (!t->tables)
4922                         continue;
4923
4924                 for (j = 0; j < p->n_tables; j++) {
4925                         struct table_runtime *r = &t->tables[j];
4926
4927                         free(r->mailbox);
4928                 }
4929
4930                 free(t->tables);
4931                 t->tables = NULL;
4932         }
4933 }
4934
4935 static void
4936 table_free(struct rte_swx_pipeline *p)
4937 {
4938         table_build_free(p);
4939
4940         /* Tables. */
4941         for ( ; ; ) {
4942                 struct table *elem;
4943
4944                 elem = TAILQ_FIRST(&p->tables);
4945                 if (!elem)
4946                         break;
4947
4948                 TAILQ_REMOVE(&p->tables, elem, node);
4949                 free(elem->fields);
4950                 free(elem->actions);
4951                 free(elem->default_action_data);
4952                 free(elem);
4953         }
4954
4955         /* Table types. */
4956         for ( ; ; ) {
4957                 struct table_type *elem;
4958
4959                 elem = TAILQ_FIRST(&p->table_types);
4960                 if (!elem)
4961                         break;
4962
4963                 TAILQ_REMOVE(&p->table_types, elem, node);
4964                 free(elem);
4965         }
4966 }
4967
4968 /*
4969  * Pipeline.
4970  */
4971 int
4972 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
4973 {
4974         struct rte_swx_pipeline *pipeline;
4975
4976         /* Check input parameters. */
4977         CHECK(p, EINVAL);
4978
4979         /* Memory allocation. */
4980         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
4981         CHECK(pipeline, ENOMEM);
4982
4983         /* Initialization. */
4984         TAILQ_INIT(&pipeline->struct_types);
4985         TAILQ_INIT(&pipeline->port_in_types);
4986         TAILQ_INIT(&pipeline->ports_in);
4987         TAILQ_INIT(&pipeline->port_out_types);
4988         TAILQ_INIT(&pipeline->ports_out);
4989         TAILQ_INIT(&pipeline->extern_types);
4990         TAILQ_INIT(&pipeline->extern_objs);
4991         TAILQ_INIT(&pipeline->extern_funcs);
4992         TAILQ_INIT(&pipeline->headers);
4993         TAILQ_INIT(&pipeline->actions);
4994         TAILQ_INIT(&pipeline->table_types);
4995         TAILQ_INIT(&pipeline->tables);
4996
4997         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
4998         pipeline->numa_node = numa_node;
4999
5000         *p = pipeline;
5001         return 0;
5002 }
5003
5004 void
5005 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
5006 {
5007         if (!p)
5008                 return;
5009
5010         free(p->instructions);
5011
5012         table_state_free(p);
5013         table_free(p);
5014         action_free(p);
5015         metadata_free(p);
5016         header_free(p);
5017         extern_func_free(p);
5018         extern_obj_free(p);
5019         port_out_free(p);
5020         port_in_free(p);
5021         struct_free(p);
5022
5023         free(p);
5024 }
5025
5026 int
5027 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
5028                                      const char **instructions,
5029                                      uint32_t n_instructions)
5030 {
5031         int err;
5032         uint32_t i;
5033
5034         err = instruction_config(p, NULL, instructions, n_instructions);
5035         if (err)
5036                 return err;
5037
5038         /* Thread instruction pointer reset. */
5039         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
5040                 struct thread *t = &p->threads[i];
5041
5042                 thread_ip_reset(p, t);
5043         }
5044
5045         return 0;
5046 }
5047
5048 int
5049 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
5050 {
5051         int status;
5052
5053         CHECK(p, EINVAL);
5054         CHECK(p->build_done == 0, EEXIST);
5055
5056         status = port_in_build(p);
5057         if (status)
5058                 goto error;
5059
5060         status = port_out_build(p);
5061         if (status)
5062                 goto error;
5063
5064         status = struct_build(p);
5065         if (status)
5066                 goto error;
5067
5068         status = extern_obj_build(p);
5069         if (status)
5070                 goto error;
5071
5072         status = extern_func_build(p);
5073         if (status)
5074                 goto error;
5075
5076         status = header_build(p);
5077         if (status)
5078                 goto error;
5079
5080         status = metadata_build(p);
5081         if (status)
5082                 goto error;
5083
5084         status = action_build(p);
5085         if (status)
5086                 goto error;
5087
5088         status = table_build(p);
5089         if (status)
5090                 goto error;
5091
5092         status = table_state_build(p);
5093         if (status)
5094                 goto error;
5095
5096         p->build_done = 1;
5097         return 0;
5098
5099 error:
5100         table_state_build_free(p);
5101         table_build_free(p);
5102         action_build_free(p);
5103         metadata_build_free(p);
5104         header_build_free(p);
5105         extern_func_build_free(p);
5106         extern_obj_build_free(p);
5107         port_out_build_free(p);
5108         port_in_build_free(p);
5109         struct_build_free(p);
5110
5111         return status;
5112 }
5113
5114 void
5115 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
5116 {
5117         uint32_t i;
5118
5119         for (i = 0; i < n_instructions; i++)
5120                 instr_exec(p);
5121 }
5122
5123 /*
5124  * Control.
5125  */
5126 int
5127 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
5128                                  struct rte_swx_table_state **table_state)
5129 {
5130         if (!p || !table_state || !p->build_done)
5131                 return -EINVAL;
5132
5133         *table_state = p->table_state;
5134         return 0;
5135 }
5136
5137 int
5138 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
5139                                  struct rte_swx_table_state *table_state)
5140 {
5141         if (!p || !table_state || !p->build_done)
5142                 return -EINVAL;
5143
5144         p->table_state = table_state;
5145         return 0;
5146 }