lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9 #include <arpa/inet.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_prefetch.h>
  13 #include <rte_byteorder.h>
  14
  15 #include "rte_swx_pipeline.h"
  16 #include "rte_swx_ctl.h"
  17
  18 #define CHECK(condition, err_code)                                             \
  19 do {                                                                           \
  20         if (!(condition))                                                      \
  21                 return -(err_code);                                            \
  22 } while (0)
  23
  24 #define CHECK_NAME(name, err_code)                                             \
  25         CHECK((name) && (name)[0], err_code)
  26
  27 #ifndef TRACE_LEVEL
  28 #define TRACE_LEVEL 0
  29 #endif
  30
  31 #if TRACE_LEVEL
  32 #define TRACE(...) printf(__VA_ARGS__)
  33 #else
  34 #define TRACE(...)
  35 #endif
  36
  37 #define ntoh64(x) rte_be_to_cpu_64(x)
  38 #define hton64(x) rte_cpu_to_be_64(x)
  39
  40 /*
  41  * Struct.
  42  */
  43 struct field {
  44         char name[RTE_SWX_NAME_SIZE];
  45         uint32_t n_bits;
  46         uint32_t offset;
  47 };
  48
  49 struct struct_type {
  50         TAILQ_ENTRY(struct_type) node;
  51         char name[RTE_SWX_NAME_SIZE];
  52         struct field *fields;
  53         uint32_t n_fields;
  54         uint32_t n_bits;
  55 };
  56
  57 TAILQ_HEAD(struct_type_tailq, struct_type);
  58
  59 /*
  60  * Input port.
  61  */
  62 struct port_in_type {
  63         TAILQ_ENTRY(port_in_type) node;
  64         char name[RTE_SWX_NAME_SIZE];
  65         struct rte_swx_port_in_ops ops;
  66 };
  67
  68 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  69
  70 struct port_in {
  71         TAILQ_ENTRY(port_in) node;
  72         struct port_in_type *type;
  73         void *obj;
  74         uint32_t id;
  75 };
  76
  77 TAILQ_HEAD(port_in_tailq, port_in);
  78
  79 struct port_in_runtime {
  80         rte_swx_port_in_pkt_rx_t pkt_rx;
  81         void *obj;
  82 };
  83
  84 /*
  85  * Output port.
  86  */
  87 struct port_out_type {
  88         TAILQ_ENTRY(port_out_type) node;
  89         char name[RTE_SWX_NAME_SIZE];
  90         struct rte_swx_port_out_ops ops;
  91 };
  92
  93 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  94
  95 struct port_out {
  96         TAILQ_ENTRY(port_out) node;
  97         struct port_out_type *type;
  98         void *obj;
  99         uint32_t id;
 100 };
 101
 102 TAILQ_HEAD(port_out_tailq, port_out);
 103
 104 struct port_out_runtime {
 105         rte_swx_port_out_pkt_tx_t pkt_tx;
 106         rte_swx_port_out_flush_t flush;
 107         void *obj;
 108 };
 109
 110 /*
 111  * Extern object.
 112  */
 113 struct extern_type_member_func {
 114         TAILQ_ENTRY(extern_type_member_func) node;
 115         char name[RTE_SWX_NAME_SIZE];
 116         rte_swx_extern_type_member_func_t func;
 117         uint32_t id;
 118 };
 119
 120 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 121
 122 struct extern_type {
 123         TAILQ_ENTRY(extern_type) node;
 124         char name[RTE_SWX_NAME_SIZE];
 125         struct struct_type *mailbox_struct_type;
 126         rte_swx_extern_type_constructor_t constructor;
 127         rte_swx_extern_type_destructor_t destructor;
 128         struct extern_type_member_func_tailq funcs;
 129         uint32_t n_funcs;
 130 };
 131
 132 TAILQ_HEAD(extern_type_tailq, extern_type);
 133
 134 struct extern_obj {
 135         TAILQ_ENTRY(extern_obj) node;
 136         char name[RTE_SWX_NAME_SIZE];
 137         struct extern_type *type;
 138         void *obj;
 139         uint32_t struct_id;
 140         uint32_t id;
 141 };
 142
 143 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 144
 145 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 146 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 147 #endif
 148
 149 struct extern_obj_runtime {
 150         void *obj;
 151         uint8_t *mailbox;
 152         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 153 };
 154
 155 /*
 156  * Extern function.
 157  */
 158 struct extern_func {
 159         TAILQ_ENTRY(extern_func) node;
 160         char name[RTE_SWX_NAME_SIZE];
 161         struct struct_type *mailbox_struct_type;
 162         rte_swx_extern_func_t func;
 163         uint32_t struct_id;
 164         uint32_t id;
 165 };
 166
 167 TAILQ_HEAD(extern_func_tailq, extern_func);
 168
 169 struct extern_func_runtime {
 170         uint8_t *mailbox;
 171         rte_swx_extern_func_t func;
 172 };
 173
 174 /*
 175  * Header.
 176  */
 177 struct header {
 178         TAILQ_ENTRY(header) node;
 179         char name[RTE_SWX_NAME_SIZE];
 180         struct struct_type *st;
 181         uint32_t struct_id;
 182         uint32_t id;
 183 };
 184
 185 TAILQ_HEAD(header_tailq, header);
 186
 187 struct header_runtime {
 188         uint8_t *ptr0;
 189 };
 190
 191 struct header_out_runtime {
 192         uint8_t *ptr0;
 193         uint8_t *ptr;
 194         uint32_t n_bytes;
 195 };
 196
 197 /*
 198  * Instruction.
 199  */
 200
 201 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 202  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 203  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 204  * when transferred to packet meta-data and in NBO when transferred to packet
 205  * headers.
 206  */
 207
 208 /* Notation conventions:
 209  *    -Header field: H = h.header.field (dst/src)
 210  *    -Meta-data field: M = m.field (dst/src)
 211  *    -Extern object mailbox field: E = e.field (dst/src)
 212  *    -Extern function mailbox field: F = f.field (dst/src)
 213  *    -Table action data field: T = t.field (src only)
 214  *    -Immediate value: I = 32-bit unsigned value (src only)
 215  */
 216
 217 enum instruction_type {
 218         /* rx m.port_in */
 219         INSTR_RX,
 220
 221         /* tx m.port_out */
 222         INSTR_TX,
 223
 224         /* extract h.header */
 225         INSTR_HDR_EXTRACT,
 226         INSTR_HDR_EXTRACT2,
 227         INSTR_HDR_EXTRACT3,
 228         INSTR_HDR_EXTRACT4,
 229         INSTR_HDR_EXTRACT5,
 230         INSTR_HDR_EXTRACT6,
 231         INSTR_HDR_EXTRACT7,
 232         INSTR_HDR_EXTRACT8,
 233
 234         /* emit h.header */
 235         INSTR_HDR_EMIT,
 236         INSTR_HDR_EMIT_TX,
 237         INSTR_HDR_EMIT2_TX,
 238         INSTR_HDR_EMIT3_TX,
 239         INSTR_HDR_EMIT4_TX,
 240         INSTR_HDR_EMIT5_TX,
 241         INSTR_HDR_EMIT6_TX,
 242         INSTR_HDR_EMIT7_TX,
 243         INSTR_HDR_EMIT8_TX,
 244
 245         /* validate h.header */
 246         INSTR_HDR_VALIDATE,
 247
 248         /* invalidate h.header */
 249         INSTR_HDR_INVALIDATE,
 250
 251         /* mov dst src
 252          * dst = src
 253          * dst = HMEF, src = HMEFTI
 254          */
 255         INSTR_MOV,   /* dst = MEF, src = MEFT */
 256         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 257         INSTR_MOV_I, /* dst = HMEF, src = I */
 258
 259         /* dma h.header t.field
 260          * memcpy(h.header, t.field, sizeof(h.header))
 261          */
 262         INSTR_DMA_HT,
 263         INSTR_DMA_HT2,
 264         INSTR_DMA_HT3,
 265         INSTR_DMA_HT4,
 266         INSTR_DMA_HT5,
 267         INSTR_DMA_HT6,
 268         INSTR_DMA_HT7,
 269         INSTR_DMA_HT8,
 270
 271         /* add dst src
 272          * dst += src
 273          * dst = HMEF, src = HMEFTI
 274          */
 275         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 276         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 277         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 278         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 279         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 280         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 281
 282         /* sub dst src
 283          * dst -= src
 284          * dst = HMEF, src = HMEFTI
 285          */
 286         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 287         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 288         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 289         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 290         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 291         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 292
 293         /* ckadd dst src
 294          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 295          * dst = H, src = {H, h.header}
 296          */
 297         INSTR_ALU_CKADD_FIELD,    /* src = H */
 298         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 299         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 300
 301         /* cksub dst src
 302          * dst = dst '- src
 303          * dst = H, src = H
 304          */
 305         INSTR_ALU_CKSUB_FIELD,
 306
 307         /* and dst src
 308          * dst &= src
 309          * dst = HMEF, src = HMEFTI
 310          */
 311         INSTR_ALU_AND,   /* dst = MEF, src = MEFT */
 312         INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 313         INSTR_ALU_AND_I, /* dst = HMEF, src = I */
 314 };
 315
 316 struct instr_operand {
 317         uint8_t struct_id;
 318         uint8_t n_bits;
 319         uint8_t offset;
 320         uint8_t pad;
 321 };
 322
 323 struct instr_io {
 324         struct {
 325                 uint8_t offset;
 326                 uint8_t n_bits;
 327                 uint8_t pad[2];
 328         } io;
 329
 330         struct {
 331                 uint8_t header_id[8];
 332                 uint8_t struct_id[8];
 333                 uint8_t n_bytes[8];
 334         } hdr;
 335 };
 336
 337 struct instr_hdr_validity {
 338         uint8_t header_id;
 339 };
 340
 341 struct instr_dst_src {
 342         struct instr_operand dst;
 343         union {
 344                 struct instr_operand src;
 345                 uint32_t src_val;
 346         };
 347 };
 348
 349 struct instr_dma {
 350         struct {
 351                 uint8_t header_id[8];
 352                 uint8_t struct_id[8];
 353         } dst;
 354
 355         struct {
 356                 uint8_t offset[8];
 357         } src;
 358
 359         uint16_t n_bytes[8];
 360 };
 361
 362 struct instruction {
 363         enum instruction_type type;
 364         union {
 365                 struct instr_io io;
 366                 struct instr_hdr_validity valid;
 367                 struct instr_dst_src mov;
 368                 struct instr_dma dma;
 369                 struct instr_dst_src alu;
 370         };
 371 };
 372
 373 struct instruction_data {
 374         char label[RTE_SWX_NAME_SIZE];
 375         char jmp_label[RTE_SWX_NAME_SIZE];
 376         uint32_t n_users; /* user = jmp instruction to this instruction. */
 377         int invalid;
 378 };
 379
 380 /*
 381  * Action.
 382  */
 383 struct action {
 384         TAILQ_ENTRY(action) node;
 385         char name[RTE_SWX_NAME_SIZE];
 386         struct struct_type *st;
 387         struct instruction *instructions;
 388         uint32_t n_instructions;
 389         uint32_t id;
 390 };
 391
 392 TAILQ_HEAD(action_tailq, action);
 393
 394 /*
 395  * Table.
 396  */
 397 struct table_type {
 398         TAILQ_ENTRY(table_type) node;
 399         char name[RTE_SWX_NAME_SIZE];
 400         enum rte_swx_table_match_type match_type;
 401         struct rte_swx_table_ops ops;
 402 };
 403
 404 TAILQ_HEAD(table_type_tailq, table_type);
 405
 406 struct match_field {
 407         enum rte_swx_table_match_type match_type;
 408         struct field *field;
 409 };
 410
 411 struct table {
 412         TAILQ_ENTRY(table) node;
 413         char name[RTE_SWX_NAME_SIZE];
 414         char args[RTE_SWX_NAME_SIZE];
 415         struct table_type *type; /* NULL when n_fields == 0. */
 416
 417         /* Match. */
 418         struct match_field *fields;
 419         uint32_t n_fields;
 420         int is_header; /* Only valid when n_fields > 0. */
 421         struct header *header; /* Only valid when n_fields > 0. */
 422
 423         /* Action. */
 424         struct action **actions;
 425         struct action *default_action;
 426         uint8_t *default_action_data;
 427         uint32_t n_actions;
 428         int default_action_is_const;
 429         uint32_t action_data_size_max;
 430
 431         uint32_t size;
 432         uint32_t id;
 433 };
 434
 435 TAILQ_HEAD(table_tailq, table);
 436
 437 struct table_runtime {
 438         rte_swx_table_lookup_t func;
 439         void *mailbox;
 440         uint8_t **key;
 441 };
 442
 443 /*
 444  * Pipeline.
 445  */
 446 struct thread {
 447         /* Packet. */
 448         struct rte_swx_pkt pkt;
 449         uint8_t *ptr;
 450
 451         /* Structures. */
 452         uint8_t **structs;
 453
 454         /* Packet headers. */
 455         struct header_runtime *headers; /* Extracted or generated headers. */
 456         struct header_out_runtime *headers_out; /* Emitted headers. */
 457         uint8_t *header_storage;
 458         uint8_t *header_out_storage;
 459         uint64_t valid_headers;
 460         uint32_t n_headers_out;
 461
 462         /* Packet meta-data. */
 463         uint8_t *metadata;
 464
 465         /* Tables. */
 466         struct table_runtime *tables;
 467         struct rte_swx_table_state *table_state;
 468         uint64_t action_id;
 469         int hit; /* 0 = Miss, 1 = Hit. */
 470
 471         /* Extern objects and functions. */
 472         struct extern_obj_runtime *extern_objs;
 473         struct extern_func_runtime *extern_funcs;
 474
 475         /* Instructions. */
 476         struct instruction *ip;
 477         struct instruction *ret;
 478 };
 479
 480 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 481 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 482 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 483
 484 #define ALU(thread, ip, operator)  \
 485 {                                                                              \
 486         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 487         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 488         uint64_t dst64 = *dst64_ptr;                                           \
 489         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 490         uint64_t dst = dst64 & dst64_mask;                                     \
 491                                                                                \
 492         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 493         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 494         uint64_t src64 = *src64_ptr;                                           \
 495         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 496         uint64_t src = src64 & src64_mask;                                     \
 497                                                                                \
 498         uint64_t result = dst operator src;                                    \
 499                                                                                \
 500         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 501 }
 502
 503 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 504
 505 #define ALU_S(thread, ip, operator)  \
 506 {                                                                              \
 507         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 508         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 509         uint64_t dst64 = *dst64_ptr;                                           \
 510         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 511         uint64_t dst = dst64 & dst64_mask;                                     \
 512                                                                                \
 513         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 514         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 515         uint64_t src64 = *src64_ptr;                                           \
 516         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 517                                                                                \
 518         uint64_t result = dst operator src;                                    \
 519                                                                                \
 520         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 521 }
 522
 523 #define ALU_MH ALU_S
 524
 525 #define ALU_HM(thread, ip, operator)  \
 526 {                                                                              \
 527         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 528         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 529         uint64_t dst64 = *dst64_ptr;                                           \
 530         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 531         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 532                                                                                \
 533         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 534         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 535         uint64_t src64 = *src64_ptr;                                           \
 536         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 537         uint64_t src = src64 & src64_mask;                                     \
 538                                                                                \
 539         uint64_t result = dst operator src;                                    \
 540         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 541                                                                                \
 542         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 543 }
 544
 545 #define ALU_HH(thread, ip, operator)  \
 546 {                                                                              \
 547         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 548         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 549         uint64_t dst64 = *dst64_ptr;                                           \
 550         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 551         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 552                                                                                \
 553         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 554         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 555         uint64_t src64 = *src64_ptr;                                           \
 556         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 557                                                                                \
 558         uint64_t result = dst operator src;                                    \
 559         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 560                                                                                \
 561         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 562 }
 563
 564 #else
 565
 566 #define ALU_S ALU
 567 #define ALU_MH ALU
 568 #define ALU_HM ALU
 569 #define ALU_HH ALU
 570
 571 #endif
 572
 573 #define ALU_I(thread, ip, operator)  \
 574 {                                                                              \
 575         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 576         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 577         uint64_t dst64 = *dst64_ptr;                                           \
 578         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 579         uint64_t dst = dst64 & dst64_mask;                                     \
 580                                                                                \
 581         uint64_t src = (ip)->alu.src_val;                                      \
 582                                                                                \
 583         uint64_t result = dst operator src;                                    \
 584                                                                                \
 585         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 586 }
 587
 588 #define ALU_MI ALU_I
 589
 590 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 591
 592 #define ALU_HI(thread, ip, operator)  \
 593 {                                                                              \
 594         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 595         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 596         uint64_t dst64 = *dst64_ptr;                                           \
 597         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 598         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 599                                                                                \
 600         uint64_t src = (ip)->alu.src_val;                                      \
 601                                                                                \
 602         uint64_t result = dst operator src;                                    \
 603         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 604                                                                                \
 605         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 606 }
 607
 608 #else
 609
 610 #define ALU_HI ALU_I
 611
 612 #endif
 613
 614 #define MOV(thread, ip)  \
 615 {                                                                              \
 616         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 617         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 618         uint64_t dst64 = *dst64_ptr;                                           \
 619         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 620                                                                                \
 621         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 622         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 623         uint64_t src64 = *src64_ptr;                                           \
 624         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
 625         uint64_t src = src64 & src64_mask;                                     \
 626                                                                                \
 627         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 628 }
 629
 630 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 631
 632 #define MOV_S(thread, ip)  \
 633 {                                                                              \
 634         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 635         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 636         uint64_t dst64 = *dst64_ptr;                                           \
 637         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 638                                                                                \
 639         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 640         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 641         uint64_t src64 = *src64_ptr;                                           \
 642         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
 643                                                                                \
 644         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 645 }
 646
 647 #else
 648
 649 #define MOV_S MOV
 650
 651 #endif
 652
 653 #define MOV_I(thread, ip)  \
 654 {                                                                              \
 655         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 656         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 657         uint64_t dst64 = *dst64_ptr;                                           \
 658         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 659                                                                                \
 660         uint64_t src = (ip)->mov.src_val;                                      \
 661                                                                                \
 662         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 663 }
 664
 665 #define METADATA_READ(thread, offset, n_bits)                                  \
 666 ({                                                                             \
 667         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 668         uint64_t m64 = *m64_ptr;                                               \
 669         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 670         (m64 & m64_mask);                                                      \
 671 })
 672
 673 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
 674 {                                                                              \
 675         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 676         uint64_t m64 = *m64_ptr;                                               \
 677         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 678                                                                                \
 679         uint64_t m_new = value;                                                \
 680                                                                                \
 681         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
 682 }
 683
 684 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 685 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 686 #endif
 687
 688 struct rte_swx_pipeline {
 689         struct struct_type_tailq struct_types;
 690         struct port_in_type_tailq port_in_types;
 691         struct port_in_tailq ports_in;
 692         struct port_out_type_tailq port_out_types;
 693         struct port_out_tailq ports_out;
 694         struct extern_type_tailq extern_types;
 695         struct extern_obj_tailq extern_objs;
 696         struct extern_func_tailq extern_funcs;
 697         struct header_tailq headers;
 698         struct struct_type *metadata_st;
 699         uint32_t metadata_struct_id;
 700         struct action_tailq actions;
 701         struct table_type_tailq table_types;
 702         struct table_tailq tables;
 703
 704         struct port_in_runtime *in;
 705         struct port_out_runtime *out;
 706         struct instruction **action_instructions;
 707         struct rte_swx_table_state *table_state;
 708         struct instruction *instructions;
 709         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 710
 711         uint32_t n_structs;
 712         uint32_t n_ports_in;
 713         uint32_t n_ports_out;
 714         uint32_t n_extern_objs;
 715         uint32_t n_extern_funcs;
 716         uint32_t n_actions;
 717         uint32_t n_tables;
 718         uint32_t n_headers;
 719         uint32_t thread_id;
 720         uint32_t port_id;
 721         uint32_t n_instructions;
 722         int build_done;
 723         int numa_node;
 724 };
 725
 726 /*
 727  * Struct.
 728  */
 729 static struct struct_type *
 730 struct_type_find(struct rte_swx_pipeline *p, const char *name)
 731 {
 732         struct struct_type *elem;
 733
 734         TAILQ_FOREACH(elem, &p->struct_types, node)
 735                 if (strcmp(elem->name, name) == 0)
 736                         return elem;
 737
 738         return NULL;
 739 }
 740
 741 static struct field *
 742 struct_type_field_find(struct struct_type *st, const char *name)
 743 {
 744         uint32_t i;
 745
 746         for (i = 0; i < st->n_fields; i++) {
 747                 struct field *f = &st->fields[i];
 748
 749                 if (strcmp(f->name, name) == 0)
 750                         return f;
 751         }
 752
 753         return NULL;
 754 }
 755
 756 int
 757 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 758                                       const char *name,
 759                                       struct rte_swx_field_params *fields,
 760                                       uint32_t n_fields)
 761 {
 762         struct struct_type *st;
 763         uint32_t i;
 764
 765         CHECK(p, EINVAL);
 766         CHECK_NAME(name, EINVAL);
 767         CHECK(fields, EINVAL);
 768         CHECK(n_fields, EINVAL);
 769
 770         for (i = 0; i < n_fields; i++) {
 771                 struct rte_swx_field_params *f = &fields[i];
 772                 uint32_t j;
 773
 774                 CHECK_NAME(f->name, EINVAL);
 775                 CHECK(f->n_bits, EINVAL);
 776                 CHECK(f->n_bits <= 64, EINVAL);
 777                 CHECK((f->n_bits & 7) == 0, EINVAL);
 778
 779                 for (j = 0; j < i; j++) {
 780                         struct rte_swx_field_params *f_prev = &fields[j];
 781
 782                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 783                 }
 784         }
 785
 786         CHECK(!struct_type_find(p, name), EEXIST);
 787
 788         /* Node allocation. */
 789         st = calloc(1, sizeof(struct struct_type));
 790         CHECK(st, ENOMEM);
 791
 792         st->fields = calloc(n_fields, sizeof(struct field));
 793         if (!st->fields) {
 794                 free(st);
 795                 CHECK(0, ENOMEM);
 796         }
 797
 798         /* Node initialization. */
 799         strcpy(st->name, name);
 800         for (i = 0; i < n_fields; i++) {
 801                 struct field *dst = &st->fields[i];
 802                 struct rte_swx_field_params *src = &fields[i];
 803
 804                 strcpy(dst->name, src->name);
 805                 dst->n_bits = src->n_bits;
 806                 dst->offset = st->n_bits;
 807
 808                 st->n_bits += src->n_bits;
 809         }
 810         st->n_fields = n_fields;
 811
 812         /* Node add to tailq. */
 813         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 814
 815         return 0;
 816 }
 817
 818 static int
 819 struct_build(struct rte_swx_pipeline *p)
 820 {
 821         uint32_t i;
 822
 823         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 824                 struct thread *t = &p->threads[i];
 825
 826                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 827                 CHECK(t->structs, ENOMEM);
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 struct_build_free(struct rte_swx_pipeline *p)
 835 {
 836         uint32_t i;
 837
 838         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 839                 struct thread *t = &p->threads[i];
 840
 841                 free(t->structs);
 842                 t->structs = NULL;
 843         }
 844 }
 845
 846 static void
 847 struct_free(struct rte_swx_pipeline *p)
 848 {
 849         struct_build_free(p);
 850
 851         /* Struct types. */
 852         for ( ; ; ) {
 853                 struct struct_type *elem;
 854
 855                 elem = TAILQ_FIRST(&p->struct_types);
 856                 if (!elem)
 857                         break;
 858
 859                 TAILQ_REMOVE(&p->struct_types, elem, node);
 860                 free(elem->fields);
 861                 free(elem);
 862         }
 863 }
 864
 865 /*
 866  * Input port.
 867  */
 868 static struct port_in_type *
 869 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 870 {
 871         struct port_in_type *elem;
 872
 873         if (!name)
 874                 return NULL;
 875
 876         TAILQ_FOREACH(elem, &p->port_in_types, node)
 877                 if (strcmp(elem->name, name) == 0)
 878                         return elem;
 879
 880         return NULL;
 881 }
 882
 883 int
 884 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 885                                        const char *name,
 886                                        struct rte_swx_port_in_ops *ops)
 887 {
 888         struct port_in_type *elem;
 889
 890         CHECK(p, EINVAL);
 891         CHECK_NAME(name, EINVAL);
 892         CHECK(ops, EINVAL);
 893         CHECK(ops->create, EINVAL);
 894         CHECK(ops->free, EINVAL);
 895         CHECK(ops->pkt_rx, EINVAL);
 896         CHECK(ops->stats_read, EINVAL);
 897
 898         CHECK(!port_in_type_find(p, name), EEXIST);
 899
 900         /* Node allocation. */
 901         elem = calloc(1, sizeof(struct port_in_type));
 902         CHECK(elem, ENOMEM);
 903
 904         /* Node initialization. */
 905         strcpy(elem->name, name);
 906         memcpy(&elem->ops, ops, sizeof(*ops));
 907
 908         /* Node add to tailq. */
 909         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 910
 911         return 0;
 912 }
 913
 914 static struct port_in *
 915 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 916 {
 917         struct port_in *port;
 918
 919         TAILQ_FOREACH(port, &p->ports_in, node)
 920                 if (port->id == port_id)
 921                         return port;
 922
 923         return NULL;
 924 }
 925
 926 int
 927 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 928                                 uint32_t port_id,
 929                                 const char *port_type_name,
 930                                 void *args)
 931 {
 932         struct port_in_type *type = NULL;
 933         struct port_in *port = NULL;
 934         void *obj = NULL;
 935
 936         CHECK(p, EINVAL);
 937
 938         CHECK(!port_in_find(p, port_id), EINVAL);
 939
 940         CHECK_NAME(port_type_name, EINVAL);
 941         type = port_in_type_find(p, port_type_name);
 942         CHECK(type, EINVAL);
 943
 944         obj = type->ops.create(args);
 945         CHECK(obj, ENODEV);
 946
 947         /* Node allocation. */
 948         port = calloc(1, sizeof(struct port_in));
 949         CHECK(port, ENOMEM);
 950
 951         /* Node initialization. */
 952         port->type = type;
 953         port->obj = obj;
 954         port->id = port_id;
 955
 956         /* Node add to tailq. */
 957         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 958         if (p->n_ports_in < port_id + 1)
 959                 p->n_ports_in = port_id + 1;
 960
 961         return 0;
 962 }
 963
 964 static int
 965 port_in_build(struct rte_swx_pipeline *p)
 966 {
 967         struct port_in *port;
 968         uint32_t i;
 969
 970         CHECK(p->n_ports_in, EINVAL);
 971         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 972
 973         for (i = 0; i < p->n_ports_in; i++)
 974                 CHECK(port_in_find(p, i), EINVAL);
 975
 976         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 977         CHECK(p->in, ENOMEM);
 978
 979         TAILQ_FOREACH(port, &p->ports_in, node) {
 980                 struct port_in_runtime *in = &p->in[port->id];
 981
 982                 in->pkt_rx = port->type->ops.pkt_rx;
 983                 in->obj = port->obj;
 984         }
 985
 986         return 0;
 987 }
 988
 989 static void
 990 port_in_build_free(struct rte_swx_pipeline *p)
 991 {
 992         free(p->in);
 993         p->in = NULL;
 994 }
 995
 996 static void
 997 port_in_free(struct rte_swx_pipeline *p)
 998 {
 999         port_in_build_free(p);
1000
1001         /* Input ports. */
1002         for ( ; ; ) {
1003                 struct port_in *port;
1004
1005                 port = TAILQ_FIRST(&p->ports_in);
1006                 if (!port)
1007                         break;
1008
1009                 TAILQ_REMOVE(&p->ports_in, port, node);
1010                 port->type->ops.free(port->obj);
1011                 free(port);
1012         }
1013
1014         /* Input port types. */
1015         for ( ; ; ) {
1016                 struct port_in_type *elem;
1017
1018                 elem = TAILQ_FIRST(&p->port_in_types);
1019                 if (!elem)
1020                         break;
1021
1022                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1023                 free(elem);
1024         }
1025 }
1026
1027 /*
1028  * Output port.
1029  */
1030 static struct port_out_type *
1031 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1032 {
1033         struct port_out_type *elem;
1034
1035         if (!name)
1036                 return NULL;
1037
1038         TAILQ_FOREACH(elem, &p->port_out_types, node)
1039                 if (!strcmp(elem->name, name))
1040                         return elem;
1041
1042         return NULL;
1043 }
1044
1045 int
1046 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1047                                         const char *name,
1048                                         struct rte_swx_port_out_ops *ops)
1049 {
1050         struct port_out_type *elem;
1051
1052         CHECK(p, EINVAL);
1053         CHECK_NAME(name, EINVAL);
1054         CHECK(ops, EINVAL);
1055         CHECK(ops->create, EINVAL);
1056         CHECK(ops->free, EINVAL);
1057         CHECK(ops->pkt_tx, EINVAL);
1058         CHECK(ops->stats_read, EINVAL);
1059
1060         CHECK(!port_out_type_find(p, name), EEXIST);
1061
1062         /* Node allocation. */
1063         elem = calloc(1, sizeof(struct port_out_type));
1064         CHECK(elem, ENOMEM);
1065
1066         /* Node initialization. */
1067         strcpy(elem->name, name);
1068         memcpy(&elem->ops, ops, sizeof(*ops));
1069
1070         /* Node add to tailq. */
1071         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1072
1073         return 0;
1074 }
1075
1076 static struct port_out *
1077 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1078 {
1079         struct port_out *port;
1080
1081         TAILQ_FOREACH(port, &p->ports_out, node)
1082                 if (port->id == port_id)
1083                         return port;
1084
1085         return NULL;
1086 }
1087
1088 int
1089 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1090                                  uint32_t port_id,
1091                                  const char *port_type_name,
1092                                  void *args)
1093 {
1094         struct port_out_type *type = NULL;
1095         struct port_out *port = NULL;
1096         void *obj = NULL;
1097
1098         CHECK(p, EINVAL);
1099
1100         CHECK(!port_out_find(p, port_id), EINVAL);
1101
1102         CHECK_NAME(port_type_name, EINVAL);
1103         type = port_out_type_find(p, port_type_name);
1104         CHECK(type, EINVAL);
1105
1106         obj = type->ops.create(args);
1107         CHECK(obj, ENODEV);
1108
1109         /* Node allocation. */
1110         port = calloc(1, sizeof(struct port_out));
1111         CHECK(port, ENOMEM);
1112
1113         /* Node initialization. */
1114         port->type = type;
1115         port->obj = obj;
1116         port->id = port_id;
1117
1118         /* Node add to tailq. */
1119         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1120         if (p->n_ports_out < port_id + 1)
1121                 p->n_ports_out = port_id + 1;
1122
1123         return 0;
1124 }
1125
1126 static int
1127 port_out_build(struct rte_swx_pipeline *p)
1128 {
1129         struct port_out *port;
1130         uint32_t i;
1131
1132         CHECK(p->n_ports_out, EINVAL);
1133
1134         for (i = 0; i < p->n_ports_out; i++)
1135                 CHECK(port_out_find(p, i), EINVAL);
1136
1137         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1138         CHECK(p->out, ENOMEM);
1139
1140         TAILQ_FOREACH(port, &p->ports_out, node) {
1141                 struct port_out_runtime *out = &p->out[port->id];
1142
1143                 out->pkt_tx = port->type->ops.pkt_tx;
1144                 out->flush = port->type->ops.flush;
1145                 out->obj = port->obj;
1146         }
1147
1148         return 0;
1149 }
1150
1151 static void
1152 port_out_build_free(struct rte_swx_pipeline *p)
1153 {
1154         free(p->out);
1155         p->out = NULL;
1156 }
1157
1158 static void
1159 port_out_free(struct rte_swx_pipeline *p)
1160 {
1161         port_out_build_free(p);
1162
1163         /* Output ports. */
1164         for ( ; ; ) {
1165                 struct port_out *port;
1166
1167                 port = TAILQ_FIRST(&p->ports_out);
1168                 if (!port)
1169                         break;
1170
1171                 TAILQ_REMOVE(&p->ports_out, port, node);
1172                 port->type->ops.free(port->obj);
1173                 free(port);
1174         }
1175
1176         /* Output port types. */
1177         for ( ; ; ) {
1178                 struct port_out_type *elem;
1179
1180                 elem = TAILQ_FIRST(&p->port_out_types);
1181                 if (!elem)
1182                         break;
1183
1184                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1185                 free(elem);
1186         }
1187 }
1188
1189 /*
1190  * Extern object.
1191  */
1192 static struct extern_type *
1193 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1194 {
1195         struct extern_type *elem;
1196
1197         TAILQ_FOREACH(elem, &p->extern_types, node)
1198                 if (strcmp(elem->name, name) == 0)
1199                         return elem;
1200
1201         return NULL;
1202 }
1203
1204 static struct extern_type_member_func *
1205 extern_type_member_func_find(struct extern_type *type, const char *name)
1206 {
1207         struct extern_type_member_func *elem;
1208
1209         TAILQ_FOREACH(elem, &type->funcs, node)
1210                 if (strcmp(elem->name, name) == 0)
1211                         return elem;
1212
1213         return NULL;
1214 }
1215
1216 static struct extern_obj *
1217 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1218 {
1219         struct extern_obj *elem;
1220
1221         TAILQ_FOREACH(elem, &p->extern_objs, node)
1222                 if (strcmp(elem->name, name) == 0)
1223                         return elem;
1224
1225         return NULL;
1226 }
1227
1228 static struct field *
1229 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1230                                const char *name,
1231                                struct extern_obj **object)
1232 {
1233         struct extern_obj *obj;
1234         struct field *f;
1235         char *obj_name, *field_name;
1236
1237         if ((name[0] != 'e') || (name[1] != '.'))
1238                 return NULL;
1239
1240         obj_name = strdup(&name[2]);
1241         if (!obj_name)
1242                 return NULL;
1243
1244         field_name = strchr(obj_name, '.');
1245         if (!field_name) {
1246                 free(obj_name);
1247                 return NULL;
1248         }
1249
1250         *field_name = 0;
1251         field_name++;
1252
1253         obj = extern_obj_find(p, obj_name);
1254         if (!obj) {
1255                 free(obj_name);
1256                 return NULL;
1257         }
1258
1259         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1260         if (!f) {
1261                 free(obj_name);
1262                 return NULL;
1263         }
1264
1265         if (object)
1266                 *object = obj;
1267
1268         free(obj_name);
1269         return f;
1270 }
1271
1272 int
1273 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1274         const char *name,
1275         const char *mailbox_struct_type_name,
1276         rte_swx_extern_type_constructor_t constructor,
1277         rte_swx_extern_type_destructor_t destructor)
1278 {
1279         struct extern_type *elem;
1280         struct struct_type *mailbox_struct_type;
1281
1282         CHECK(p, EINVAL);
1283
1284         CHECK_NAME(name, EINVAL);
1285         CHECK(!extern_type_find(p, name), EEXIST);
1286
1287         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1288         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1289         CHECK(mailbox_struct_type, EINVAL);
1290
1291         CHECK(constructor, EINVAL);
1292         CHECK(destructor, EINVAL);
1293
1294         /* Node allocation. */
1295         elem = calloc(1, sizeof(struct extern_type));
1296         CHECK(elem, ENOMEM);
1297
1298         /* Node initialization. */
1299         strcpy(elem->name, name);
1300         elem->mailbox_struct_type = mailbox_struct_type;
1301         elem->constructor = constructor;
1302         elem->destructor = destructor;
1303         TAILQ_INIT(&elem->funcs);
1304
1305         /* Node add to tailq. */
1306         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1307
1308         return 0;
1309 }
1310
1311 int
1312 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1313         const char *extern_type_name,
1314         const char *name,
1315         rte_swx_extern_type_member_func_t member_func)
1316 {
1317         struct extern_type *type;
1318         struct extern_type_member_func *type_member;
1319
1320         CHECK(p, EINVAL);
1321
1322         CHECK(extern_type_name, EINVAL);
1323         type = extern_type_find(p, extern_type_name);
1324         CHECK(type, EINVAL);
1325         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1326
1327         CHECK(name, EINVAL);
1328         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1329
1330         CHECK(member_func, EINVAL);
1331
1332         /* Node allocation. */
1333         type_member = calloc(1, sizeof(struct extern_type_member_func));
1334         CHECK(type_member, ENOMEM);
1335
1336         /* Node initialization. */
1337         strcpy(type_member->name, name);
1338         type_member->func = member_func;
1339         type_member->id = type->n_funcs;
1340
1341         /* Node add to tailq. */
1342         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1343         type->n_funcs++;
1344
1345         return 0;
1346 }
1347
1348 int
1349 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1350                                       const char *extern_type_name,
1351                                       const char *name,
1352                                       const char *args)
1353 {
1354         struct extern_type *type;
1355         struct extern_obj *obj;
1356         void *obj_handle;
1357
1358         CHECK(p, EINVAL);
1359
1360         CHECK_NAME(extern_type_name, EINVAL);
1361         type = extern_type_find(p, extern_type_name);
1362         CHECK(type, EINVAL);
1363
1364         CHECK_NAME(name, EINVAL);
1365         CHECK(!extern_obj_find(p, name), EEXIST);
1366
1367         /* Node allocation. */
1368         obj = calloc(1, sizeof(struct extern_obj));
1369         CHECK(obj, ENOMEM);
1370
1371         /* Object construction. */
1372         obj_handle = type->constructor(args);
1373         if (!obj_handle) {
1374                 free(obj);
1375                 CHECK(0, ENODEV);
1376         }
1377
1378         /* Node initialization. */
1379         strcpy(obj->name, name);
1380         obj->type = type;
1381         obj->obj = obj_handle;
1382         obj->struct_id = p->n_structs;
1383         obj->id = p->n_extern_objs;
1384
1385         /* Node add to tailq. */
1386         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1387         p->n_extern_objs++;
1388         p->n_structs++;
1389
1390         return 0;
1391 }
1392
1393 static int
1394 extern_obj_build(struct rte_swx_pipeline *p)
1395 {
1396         uint32_t i;
1397
1398         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1399                 struct thread *t = &p->threads[i];
1400                 struct extern_obj *obj;
1401
1402                 t->extern_objs = calloc(p->n_extern_objs,
1403                                         sizeof(struct extern_obj_runtime));
1404                 CHECK(t->extern_objs, ENOMEM);
1405
1406                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1407                         struct extern_obj_runtime *r =
1408                                 &t->extern_objs[obj->id];
1409                         struct extern_type_member_func *func;
1410                         uint32_t mailbox_size =
1411                                 obj->type->mailbox_struct_type->n_bits / 8;
1412
1413                         r->obj = obj->obj;
1414
1415                         r->mailbox = calloc(1, mailbox_size);
1416                         CHECK(r->mailbox, ENOMEM);
1417
1418                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1419                                 r->funcs[func->id] = func->func;
1420
1421                         t->structs[obj->struct_id] = r->mailbox;
1422                 }
1423         }
1424
1425         return 0;
1426 }
1427
1428 static void
1429 extern_obj_build_free(struct rte_swx_pipeline *p)
1430 {
1431         uint32_t i;
1432
1433         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1434                 struct thread *t = &p->threads[i];
1435                 uint32_t j;
1436
1437                 if (!t->extern_objs)
1438                         continue;
1439
1440                 for (j = 0; j < p->n_extern_objs; j++) {
1441                         struct extern_obj_runtime *r = &t->extern_objs[j];
1442
1443                         free(r->mailbox);
1444                 }
1445
1446                 free(t->extern_objs);
1447                 t->extern_objs = NULL;
1448         }
1449 }
1450
1451 static void
1452 extern_obj_free(struct rte_swx_pipeline *p)
1453 {
1454         extern_obj_build_free(p);
1455
1456         /* Extern objects. */
1457         for ( ; ; ) {
1458                 struct extern_obj *elem;
1459
1460                 elem = TAILQ_FIRST(&p->extern_objs);
1461                 if (!elem)
1462                         break;
1463
1464                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1465                 if (elem->obj)
1466                         elem->type->destructor(elem->obj);
1467                 free(elem);
1468         }
1469
1470         /* Extern types. */
1471         for ( ; ; ) {
1472                 struct extern_type *elem;
1473
1474                 elem = TAILQ_FIRST(&p->extern_types);
1475                 if (!elem)
1476                         break;
1477
1478                 TAILQ_REMOVE(&p->extern_types, elem, node);
1479
1480                 for ( ; ; ) {
1481                         struct extern_type_member_func *func;
1482
1483                         func = TAILQ_FIRST(&elem->funcs);
1484                         if (!func)
1485                                 break;
1486
1487                         TAILQ_REMOVE(&elem->funcs, func, node);
1488                         free(func);
1489                 }
1490
1491                 free(elem);
1492         }
1493 }
1494
1495 /*
1496  * Extern function.
1497  */
1498 static struct extern_func *
1499 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1500 {
1501         struct extern_func *elem;
1502
1503         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1504                 if (strcmp(elem->name, name) == 0)
1505                         return elem;
1506
1507         return NULL;
1508 }
1509
1510 static struct field *
1511 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1512                                 const char *name,
1513                                 struct extern_func **function)
1514 {
1515         struct extern_func *func;
1516         struct field *f;
1517         char *func_name, *field_name;
1518
1519         if ((name[0] != 'f') || (name[1] != '.'))
1520                 return NULL;
1521
1522         func_name = strdup(&name[2]);
1523         if (!func_name)
1524                 return NULL;
1525
1526         field_name = strchr(func_name, '.');
1527         if (!field_name) {
1528                 free(func_name);
1529                 return NULL;
1530         }
1531
1532         *field_name = 0;
1533         field_name++;
1534
1535         func = extern_func_find(p, func_name);
1536         if (!func) {
1537                 free(func_name);
1538                 return NULL;
1539         }
1540
1541         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1542         if (!f) {
1543                 free(func_name);
1544                 return NULL;
1545         }
1546
1547         if (function)
1548                 *function = func;
1549
1550         free(func_name);
1551         return f;
1552 }
1553
1554 int
1555 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1556                                       const char *name,
1557                                       const char *mailbox_struct_type_name,
1558                                       rte_swx_extern_func_t func)
1559 {
1560         struct extern_func *f;
1561         struct struct_type *mailbox_struct_type;
1562
1563         CHECK(p, EINVAL);
1564
1565         CHECK_NAME(name, EINVAL);
1566         CHECK(!extern_func_find(p, name), EEXIST);
1567
1568         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1569         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1570         CHECK(mailbox_struct_type, EINVAL);
1571
1572         CHECK(func, EINVAL);
1573
1574         /* Node allocation. */
1575         f = calloc(1, sizeof(struct extern_func));
1576         CHECK(func, ENOMEM);
1577
1578         /* Node initialization. */
1579         strcpy(f->name, name);
1580         f->mailbox_struct_type = mailbox_struct_type;
1581         f->func = func;
1582         f->struct_id = p->n_structs;
1583         f->id = p->n_extern_funcs;
1584
1585         /* Node add to tailq. */
1586         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1587         p->n_extern_funcs++;
1588         p->n_structs++;
1589
1590         return 0;
1591 }
1592
1593 static int
1594 extern_func_build(struct rte_swx_pipeline *p)
1595 {
1596         uint32_t i;
1597
1598         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1599                 struct thread *t = &p->threads[i];
1600                 struct extern_func *func;
1601
1602                 /* Memory allocation. */
1603                 t->extern_funcs = calloc(p->n_extern_funcs,
1604                                          sizeof(struct extern_func_runtime));
1605                 CHECK(t->extern_funcs, ENOMEM);
1606
1607                 /* Extern function. */
1608                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1609                         struct extern_func_runtime *r =
1610                                 &t->extern_funcs[func->id];
1611                         uint32_t mailbox_size =
1612                                 func->mailbox_struct_type->n_bits / 8;
1613
1614                         r->func = func->func;
1615
1616                         r->mailbox = calloc(1, mailbox_size);
1617                         CHECK(r->mailbox, ENOMEM);
1618
1619                         t->structs[func->struct_id] = r->mailbox;
1620                 }
1621         }
1622
1623         return 0;
1624 }
1625
1626 static void
1627 extern_func_build_free(struct rte_swx_pipeline *p)
1628 {
1629         uint32_t i;
1630
1631         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1632                 struct thread *t = &p->threads[i];
1633                 uint32_t j;
1634
1635                 if (!t->extern_funcs)
1636                         continue;
1637
1638                 for (j = 0; j < p->n_extern_funcs; j++) {
1639                         struct extern_func_runtime *r = &t->extern_funcs[j];
1640
1641                         free(r->mailbox);
1642                 }
1643
1644                 free(t->extern_funcs);
1645                 t->extern_funcs = NULL;
1646         }
1647 }
1648
1649 static void
1650 extern_func_free(struct rte_swx_pipeline *p)
1651 {
1652         extern_func_build_free(p);
1653
1654         for ( ; ; ) {
1655                 struct extern_func *elem;
1656
1657                 elem = TAILQ_FIRST(&p->extern_funcs);
1658                 if (!elem)
1659                         break;
1660
1661                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1662                 free(elem);
1663         }
1664 }
1665
1666 /*
1667  * Header.
1668  */
1669 static struct header *
1670 header_find(struct rte_swx_pipeline *p, const char *name)
1671 {
1672         struct header *elem;
1673
1674         TAILQ_FOREACH(elem, &p->headers, node)
1675                 if (strcmp(elem->name, name) == 0)
1676                         return elem;
1677
1678         return NULL;
1679 }
1680
1681 static struct header *
1682 header_parse(struct rte_swx_pipeline *p,
1683              const char *name)
1684 {
1685         if (name[0] != 'h' || name[1] != '.')
1686                 return NULL;
1687
1688         return header_find(p, &name[2]);
1689 }
1690
1691 static struct field *
1692 header_field_parse(struct rte_swx_pipeline *p,
1693                    const char *name,
1694                    struct header **header)
1695 {
1696         struct header *h;
1697         struct field *f;
1698         char *header_name, *field_name;
1699
1700         if ((name[0] != 'h') || (name[1] != '.'))
1701                 return NULL;
1702
1703         header_name = strdup(&name[2]);
1704         if (!header_name)
1705                 return NULL;
1706
1707         field_name = strchr(header_name, '.');
1708         if (!field_name) {
1709                 free(header_name);
1710                 return NULL;
1711         }
1712
1713         *field_name = 0;
1714         field_name++;
1715
1716         h = header_find(p, header_name);
1717         if (!h) {
1718                 free(header_name);
1719                 return NULL;
1720         }
1721
1722         f = struct_type_field_find(h->st, field_name);
1723         if (!f) {
1724                 free(header_name);
1725                 return NULL;
1726         }
1727
1728         if (header)
1729                 *header = h;
1730
1731         free(header_name);
1732         return f;
1733 }
1734
1735 int
1736 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1737                                         const char *name,
1738                                         const char *struct_type_name)
1739 {
1740         struct struct_type *st;
1741         struct header *h;
1742         size_t n_headers_max;
1743
1744         CHECK(p, EINVAL);
1745         CHECK_NAME(name, EINVAL);
1746         CHECK_NAME(struct_type_name, EINVAL);
1747
1748         CHECK(!header_find(p, name), EEXIST);
1749
1750         st = struct_type_find(p, struct_type_name);
1751         CHECK(st, EINVAL);
1752
1753         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1754         CHECK(p->n_headers < n_headers_max, ENOSPC);
1755
1756         /* Node allocation. */
1757         h = calloc(1, sizeof(struct header));
1758         CHECK(h, ENOMEM);
1759
1760         /* Node initialization. */
1761         strcpy(h->name, name);
1762         h->st = st;
1763         h->struct_id = p->n_structs;
1764         h->id = p->n_headers;
1765
1766         /* Node add to tailq. */
1767         TAILQ_INSERT_TAIL(&p->headers, h, node);
1768         p->n_headers++;
1769         p->n_structs++;
1770
1771         return 0;
1772 }
1773
1774 static int
1775 header_build(struct rte_swx_pipeline *p)
1776 {
1777         struct header *h;
1778         uint32_t n_bytes = 0, i;
1779
1780         TAILQ_FOREACH(h, &p->headers, node) {
1781                 n_bytes += h->st->n_bits / 8;
1782         }
1783
1784         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1785                 struct thread *t = &p->threads[i];
1786                 uint32_t offset = 0;
1787
1788                 t->headers = calloc(p->n_headers,
1789                                     sizeof(struct header_runtime));
1790                 CHECK(t->headers, ENOMEM);
1791
1792                 t->headers_out = calloc(p->n_headers,
1793                                         sizeof(struct header_out_runtime));
1794                 CHECK(t->headers_out, ENOMEM);
1795
1796                 t->header_storage = calloc(1, n_bytes);
1797                 CHECK(t->header_storage, ENOMEM);
1798
1799                 t->header_out_storage = calloc(1, n_bytes);
1800                 CHECK(t->header_out_storage, ENOMEM);
1801
1802                 TAILQ_FOREACH(h, &p->headers, node) {
1803                         uint8_t *header_storage;
1804
1805                         header_storage = &t->header_storage[offset];
1806                         offset += h->st->n_bits / 8;
1807
1808                         t->headers[h->id].ptr0 = header_storage;
1809                         t->structs[h->struct_id] = header_storage;
1810                 }
1811         }
1812
1813         return 0;
1814 }
1815
1816 static void
1817 header_build_free(struct rte_swx_pipeline *p)
1818 {
1819         uint32_t i;
1820
1821         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1822                 struct thread *t = &p->threads[i];
1823
1824                 free(t->headers_out);
1825                 t->headers_out = NULL;
1826
1827                 free(t->headers);
1828                 t->headers = NULL;
1829
1830                 free(t->header_out_storage);
1831                 t->header_out_storage = NULL;
1832
1833                 free(t->header_storage);
1834                 t->header_storage = NULL;
1835         }
1836 }
1837
1838 static void
1839 header_free(struct rte_swx_pipeline *p)
1840 {
1841         header_build_free(p);
1842
1843         for ( ; ; ) {
1844                 struct header *elem;
1845
1846                 elem = TAILQ_FIRST(&p->headers);
1847                 if (!elem)
1848                         break;
1849
1850                 TAILQ_REMOVE(&p->headers, elem, node);
1851                 free(elem);
1852         }
1853 }
1854
1855 /*
1856  * Meta-data.
1857  */
1858 static struct field *
1859 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1860 {
1861         if (!p->metadata_st)
1862                 return NULL;
1863
1864         if (name[0] != 'm' || name[1] != '.')
1865                 return NULL;
1866
1867         return struct_type_field_find(p->metadata_st, &name[2]);
1868 }
1869
1870 int
1871 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1872                                           const char *struct_type_name)
1873 {
1874         struct struct_type *st = NULL;
1875
1876         CHECK(p, EINVAL);
1877
1878         CHECK_NAME(struct_type_name, EINVAL);
1879         st  = struct_type_find(p, struct_type_name);
1880         CHECK(st, EINVAL);
1881         CHECK(!p->metadata_st, EINVAL);
1882
1883         p->metadata_st = st;
1884         p->metadata_struct_id = p->n_structs;
1885
1886         p->n_structs++;
1887
1888         return 0;
1889 }
1890
1891 static int
1892 metadata_build(struct rte_swx_pipeline *p)
1893 {
1894         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1895         uint32_t i;
1896
1897         /* Thread-level initialization. */
1898         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1899                 struct thread *t = &p->threads[i];
1900                 uint8_t *metadata;
1901
1902                 metadata = calloc(1, n_bytes);
1903                 CHECK(metadata, ENOMEM);
1904
1905                 t->metadata = metadata;
1906                 t->structs[p->metadata_struct_id] = metadata;
1907         }
1908
1909         return 0;
1910 }
1911
1912 static void
1913 metadata_build_free(struct rte_swx_pipeline *p)
1914 {
1915         uint32_t i;
1916
1917         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1918                 struct thread *t = &p->threads[i];
1919
1920                 free(t->metadata);
1921                 t->metadata = NULL;
1922         }
1923 }
1924
1925 static void
1926 metadata_free(struct rte_swx_pipeline *p)
1927 {
1928         metadata_build_free(p);
1929 }
1930
1931 /*
1932  * Instruction.
1933  */
1934 static struct field *
1935 action_field_parse(struct action *action, const char *name);
1936
1937 static struct field *
1938 struct_field_parse(struct rte_swx_pipeline *p,
1939                    struct action *action,
1940                    const char *name,
1941                    uint32_t *struct_id)
1942 {
1943         struct field *f;
1944
1945         switch (name[0]) {
1946         case 'h':
1947         {
1948                 struct header *header;
1949
1950                 f = header_field_parse(p, name, &header);
1951                 if (!f)
1952                         return NULL;
1953
1954                 *struct_id = header->struct_id;
1955                 return f;
1956         }
1957
1958         case 'm':
1959         {
1960                 f = metadata_field_parse(p, name);
1961                 if (!f)
1962                         return NULL;
1963
1964                 *struct_id = p->metadata_struct_id;
1965                 return f;
1966         }
1967
1968         case 't':
1969         {
1970                 if (!action)
1971                         return NULL;
1972
1973                 f = action_field_parse(action, name);
1974                 if (!f)
1975                         return NULL;
1976
1977                 *struct_id = 0;
1978                 return f;
1979         }
1980
1981         case 'e':
1982         {
1983                 struct extern_obj *obj;
1984
1985                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1986                 if (!f)
1987                         return NULL;
1988
1989                 *struct_id = obj->struct_id;
1990                 return f;
1991         }
1992
1993         case 'f':
1994         {
1995                 struct extern_func *func;
1996
1997                 f = extern_func_mailbox_field_parse(p, name, &func);
1998                 if (!f)
1999                         return NULL;
2000
2001                 *struct_id = func->struct_id;
2002                 return f;
2003         }
2004
2005         default:
2006                 return NULL;
2007         }
2008 }
2009
2010 static inline void
2011 pipeline_port_inc(struct rte_swx_pipeline *p)
2012 {
2013         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2014 }
2015
2016 static inline void
2017 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2018 {
2019         t->ip = p->instructions;
2020 }
2021
2022 static inline void
2023 thread_ip_inc(struct rte_swx_pipeline *p);
2024
2025 static inline void
2026 thread_ip_inc(struct rte_swx_pipeline *p)
2027 {
2028         struct thread *t = &p->threads[p->thread_id];
2029
2030         t->ip++;
2031 }
2032
2033 static inline void
2034 thread_ip_inc_cond(struct thread *t, int cond)
2035 {
2036         t->ip += cond;
2037 }
2038
2039 static inline void
2040 thread_yield(struct rte_swx_pipeline *p)
2041 {
2042         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2043 }
2044
2045 /*
2046  * rx.
2047  */
2048 static int
2049 instr_rx_translate(struct rte_swx_pipeline *p,
2050                    struct action *action,
2051                    char **tokens,
2052                    int n_tokens,
2053                    struct instruction *instr,
2054                    struct instruction_data *data __rte_unused)
2055 {
2056         struct field *f;
2057
2058         CHECK(!action, EINVAL);
2059         CHECK(n_tokens == 2, EINVAL);
2060
2061         f = metadata_field_parse(p, tokens[1]);
2062         CHECK(f, EINVAL);
2063
2064         instr->type = INSTR_RX;
2065         instr->io.io.offset = f->offset / 8;
2066         instr->io.io.n_bits = f->n_bits;
2067         return 0;
2068 }
2069
2070 static inline void
2071 instr_rx_exec(struct rte_swx_pipeline *p);
2072
2073 static inline void
2074 instr_rx_exec(struct rte_swx_pipeline *p)
2075 {
2076         struct thread *t = &p->threads[p->thread_id];
2077         struct instruction *ip = t->ip;
2078         struct port_in_runtime *port = &p->in[p->port_id];
2079         struct rte_swx_pkt *pkt = &t->pkt;
2080         int pkt_received;
2081
2082         /* Packet. */
2083         pkt_received = port->pkt_rx(port->obj, pkt);
2084         t->ptr = &pkt->pkt[pkt->offset];
2085         rte_prefetch0(t->ptr);
2086
2087         TRACE("[Thread %2u] rx %s from port %u\n",
2088               p->thread_id,
2089               pkt_received ? "1 pkt" : "0 pkts",
2090               p->port_id);
2091
2092         /* Headers. */
2093         t->valid_headers = 0;
2094         t->n_headers_out = 0;
2095
2096         /* Meta-data. */
2097         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2098
2099         /* Tables. */
2100         t->table_state = p->table_state;
2101
2102         /* Thread. */
2103         pipeline_port_inc(p);
2104         thread_ip_inc_cond(t, pkt_received);
2105         thread_yield(p);
2106 }
2107
2108 /*
2109  * tx.
2110  */
2111 static int
2112 instr_tx_translate(struct rte_swx_pipeline *p,
2113                    struct action *action __rte_unused,
2114                    char **tokens,
2115                    int n_tokens,
2116                    struct instruction *instr,
2117                    struct instruction_data *data __rte_unused)
2118 {
2119         struct field *f;
2120
2121         CHECK(n_tokens == 2, EINVAL);
2122
2123         f = metadata_field_parse(p, tokens[1]);
2124         CHECK(f, EINVAL);
2125
2126         instr->type = INSTR_TX;
2127         instr->io.io.offset = f->offset / 8;
2128         instr->io.io.n_bits = f->n_bits;
2129         return 0;
2130 }
2131
2132 static inline void
2133 emit_handler(struct thread *t)
2134 {
2135         struct header_out_runtime *h0 = &t->headers_out[0];
2136         struct header_out_runtime *h1 = &t->headers_out[1];
2137         uint32_t offset = 0, i;
2138
2139         /* No header change or header decapsulation. */
2140         if ((t->n_headers_out == 1) &&
2141             (h0->ptr + h0->n_bytes == t->ptr)) {
2142                 TRACE("Emit handler: no header change or header decap.\n");
2143
2144                 t->pkt.offset -= h0->n_bytes;
2145                 t->pkt.length += h0->n_bytes;
2146
2147                 return;
2148         }
2149
2150         /* Header encapsulation (optionally, with prior header decasulation). */
2151         if ((t->n_headers_out == 2) &&
2152             (h1->ptr + h1->n_bytes == t->ptr) &&
2153             (h0->ptr == h0->ptr0)) {
2154                 uint32_t offset;
2155
2156                 TRACE("Emit handler: header encapsulation.\n");
2157
2158                 offset = h0->n_bytes + h1->n_bytes;
2159                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2160                 t->pkt.offset -= offset;
2161                 t->pkt.length += offset;
2162
2163                 return;
2164         }
2165
2166         /* Header insertion. */
2167         /* TBD */
2168
2169         /* Header extraction. */
2170         /* TBD */
2171
2172         /* For any other case. */
2173         TRACE("Emit handler: complex case.\n");
2174
2175         for (i = 0; i < t->n_headers_out; i++) {
2176                 struct header_out_runtime *h = &t->headers_out[i];
2177
2178                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2179                 offset += h->n_bytes;
2180         }
2181
2182         if (offset) {
2183                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2184                 t->pkt.offset -= offset;
2185                 t->pkt.length += offset;
2186         }
2187 }
2188
2189 static inline void
2190 instr_tx_exec(struct rte_swx_pipeline *p);
2191
2192 static inline void
2193 instr_tx_exec(struct rte_swx_pipeline *p)
2194 {
2195         struct thread *t = &p->threads[p->thread_id];
2196         struct instruction *ip = t->ip;
2197         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2198         struct port_out_runtime *port = &p->out[port_id];
2199         struct rte_swx_pkt *pkt = &t->pkt;
2200
2201         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2202               p->thread_id,
2203               (uint32_t)port_id);
2204
2205         /* Headers. */
2206         emit_handler(t);
2207
2208         /* Packet. */
2209         port->pkt_tx(port->obj, pkt);
2210
2211         /* Thread. */
2212         thread_ip_reset(p, t);
2213         instr_rx_exec(p);
2214 }
2215
2216 /*
2217  * extract.
2218  */
2219 static int
2220 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2221                             struct action *action,
2222                             char **tokens,
2223                             int n_tokens,
2224                             struct instruction *instr,
2225                             struct instruction_data *data __rte_unused)
2226 {
2227         struct header *h;
2228
2229         CHECK(!action, EINVAL);
2230         CHECK(n_tokens == 2, EINVAL);
2231
2232         h = header_parse(p, tokens[1]);
2233         CHECK(h, EINVAL);
2234
2235         instr->type = INSTR_HDR_EXTRACT;
2236         instr->io.hdr.header_id[0] = h->id;
2237         instr->io.hdr.struct_id[0] = h->struct_id;
2238         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2239         return 0;
2240 }
2241
2242 static inline void
2243 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2244
2245 static inline void
2246 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2247 {
2248         struct thread *t = &p->threads[p->thread_id];
2249         struct instruction *ip = t->ip;
2250         uint64_t valid_headers = t->valid_headers;
2251         uint8_t *ptr = t->ptr;
2252         uint32_t offset = t->pkt.offset;
2253         uint32_t length = t->pkt.length;
2254         uint32_t i;
2255
2256         for (i = 0; i < n_extract; i++) {
2257                 uint32_t header_id = ip->io.hdr.header_id[i];
2258                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2259                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2260
2261                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2262                       p->thread_id,
2263                       header_id,
2264                       n_bytes);
2265
2266                 /* Headers. */
2267                 t->structs[struct_id] = ptr;
2268                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2269
2270                 /* Packet. */
2271                 offset += n_bytes;
2272                 length -= n_bytes;
2273                 ptr += n_bytes;
2274         }
2275
2276         /* Headers. */
2277         t->valid_headers = valid_headers;
2278
2279         /* Packet. */
2280         t->pkt.offset = offset;
2281         t->pkt.length = length;
2282         t->ptr = ptr;
2283 }
2284
2285 static inline void
2286 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2287 {
2288         __instr_hdr_extract_exec(p, 1);
2289
2290         /* Thread. */
2291         thread_ip_inc(p);
2292 }
2293
2294 static inline void
2295 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2296 {
2297         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2298               p->thread_id);
2299
2300         __instr_hdr_extract_exec(p, 2);
2301
2302         /* Thread. */
2303         thread_ip_inc(p);
2304 }
2305
2306 static inline void
2307 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2308 {
2309         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2310               p->thread_id);
2311
2312         __instr_hdr_extract_exec(p, 3);
2313
2314         /* Thread. */
2315         thread_ip_inc(p);
2316 }
2317
2318 static inline void
2319 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2320 {
2321         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2322               p->thread_id);
2323
2324         __instr_hdr_extract_exec(p, 4);
2325
2326         /* Thread. */
2327         thread_ip_inc(p);
2328 }
2329
2330 static inline void
2331 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2332 {
2333         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2334               p->thread_id);
2335
2336         __instr_hdr_extract_exec(p, 5);
2337
2338         /* Thread. */
2339         thread_ip_inc(p);
2340 }
2341
2342 static inline void
2343 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2344 {
2345         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2346               p->thread_id);
2347
2348         __instr_hdr_extract_exec(p, 6);
2349
2350         /* Thread. */
2351         thread_ip_inc(p);
2352 }
2353
2354 static inline void
2355 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2356 {
2357         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2358               p->thread_id);
2359
2360         __instr_hdr_extract_exec(p, 7);
2361
2362         /* Thread. */
2363         thread_ip_inc(p);
2364 }
2365
2366 static inline void
2367 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2368 {
2369         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2370               p->thread_id);
2371
2372         __instr_hdr_extract_exec(p, 8);
2373
2374         /* Thread. */
2375         thread_ip_inc(p);
2376 }
2377
2378 /*
2379  * emit.
2380  */
2381 static int
2382 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2383                          struct action *action __rte_unused,
2384                          char **tokens,
2385                          int n_tokens,
2386                          struct instruction *instr,
2387                          struct instruction_data *data __rte_unused)
2388 {
2389         struct header *h;
2390
2391         CHECK(n_tokens == 2, EINVAL);
2392
2393         h = header_parse(p, tokens[1]);
2394         CHECK(h, EINVAL);
2395
2396         instr->type = INSTR_HDR_EMIT;
2397         instr->io.hdr.header_id[0] = h->id;
2398         instr->io.hdr.struct_id[0] = h->struct_id;
2399         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2400         return 0;
2401 }
2402
2403 static inline void
2404 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
2405
2406 static inline void
2407 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
2408 {
2409         struct thread *t = &p->threads[p->thread_id];
2410         struct instruction *ip = t->ip;
2411         uint32_t n_headers_out = t->n_headers_out;
2412         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
2413         uint8_t *ho_ptr = NULL;
2414         uint32_t ho_nbytes = 0, i;
2415
2416         for (i = 0; i < n_emit; i++) {
2417                 uint32_t header_id = ip->io.hdr.header_id[i];
2418                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2419                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2420
2421                 struct header_runtime *hi = &t->headers[header_id];
2422                 uint8_t *hi_ptr = t->structs[struct_id];
2423
2424                 TRACE("[Thread %2u]: emit header %u\n",
2425                       p->thread_id,
2426                       header_id);
2427
2428                 /* Headers. */
2429                 if (!i) {
2430                         if (!t->n_headers_out) {
2431                                 ho = &t->headers_out[0];
2432
2433                                 ho->ptr0 = hi->ptr0;
2434                                 ho->ptr = hi_ptr;
2435
2436                                 ho_ptr = hi_ptr;
2437                                 ho_nbytes = n_bytes;
2438
2439                                 n_headers_out = 1;
2440
2441                                 continue;
2442                         } else {
2443                                 ho_ptr = ho->ptr;
2444                                 ho_nbytes = ho->n_bytes;
2445                         }
2446                 }
2447
2448                 if (ho_ptr + ho_nbytes == hi_ptr) {
2449                         ho_nbytes += n_bytes;
2450                 } else {
2451                         ho->n_bytes = ho_nbytes;
2452
2453                         ho++;
2454                         ho->ptr0 = hi->ptr0;
2455                         ho->ptr = hi_ptr;
2456
2457                         ho_ptr = hi_ptr;
2458                         ho_nbytes = n_bytes;
2459
2460                         n_headers_out++;
2461                 }
2462         }
2463
2464         ho->n_bytes = ho_nbytes;
2465         t->n_headers_out = n_headers_out;
2466 }
2467
2468 static inline void
2469 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2470 {
2471         __instr_hdr_emit_exec(p, 1);
2472
2473         /* Thread. */
2474         thread_ip_inc(p);
2475 }
2476
2477 static inline void
2478 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2479 {
2480         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2481               p->thread_id);
2482
2483         __instr_hdr_emit_exec(p, 1);
2484         instr_tx_exec(p);
2485 }
2486
2487 static inline void
2488 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2489 {
2490         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2491               p->thread_id);
2492
2493         __instr_hdr_emit_exec(p, 2);
2494         instr_tx_exec(p);
2495 }
2496
2497 static inline void
2498 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2499 {
2500         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2501               p->thread_id);
2502
2503         __instr_hdr_emit_exec(p, 3);
2504         instr_tx_exec(p);
2505 }
2506
2507 static inline void
2508 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2509 {
2510         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2511               p->thread_id);
2512
2513         __instr_hdr_emit_exec(p, 4);
2514         instr_tx_exec(p);
2515 }
2516
2517 static inline void
2518 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2519 {
2520         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2521               p->thread_id);
2522
2523         __instr_hdr_emit_exec(p, 5);
2524         instr_tx_exec(p);
2525 }
2526
2527 static inline void
2528 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2529 {
2530         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2531               p->thread_id);
2532
2533         __instr_hdr_emit_exec(p, 6);
2534         instr_tx_exec(p);
2535 }
2536
2537 static inline void
2538 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2539 {
2540         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2541               p->thread_id);
2542
2543         __instr_hdr_emit_exec(p, 7);
2544         instr_tx_exec(p);
2545 }
2546
2547 static inline void
2548 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2549 {
2550         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2551               p->thread_id);
2552
2553         __instr_hdr_emit_exec(p, 8);
2554         instr_tx_exec(p);
2555 }
2556
2557 /*
2558  * validate.
2559  */
2560 static int
2561 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2562                              struct action *action __rte_unused,
2563                              char **tokens,
2564                              int n_tokens,
2565                              struct instruction *instr,
2566                              struct instruction_data *data __rte_unused)
2567 {
2568         struct header *h;
2569
2570         CHECK(n_tokens == 2, EINVAL);
2571
2572         h = header_parse(p, tokens[1]);
2573         CHECK(h, EINVAL);
2574
2575         instr->type = INSTR_HDR_VALIDATE;
2576         instr->valid.header_id = h->id;
2577         return 0;
2578 }
2579
2580 static inline void
2581 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2582 {
2583         struct thread *t = &p->threads[p->thread_id];
2584         struct instruction *ip = t->ip;
2585         uint32_t header_id = ip->valid.header_id;
2586
2587         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2588
2589         /* Headers. */
2590         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2591
2592         /* Thread. */
2593         thread_ip_inc(p);
2594 }
2595
2596 /*
2597  * invalidate.
2598  */
2599 static int
2600 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2601                                struct action *action __rte_unused,
2602                                char **tokens,
2603                                int n_tokens,
2604                                struct instruction *instr,
2605                                struct instruction_data *data __rte_unused)
2606 {
2607         struct header *h;
2608
2609         CHECK(n_tokens == 2, EINVAL);
2610
2611         h = header_parse(p, tokens[1]);
2612         CHECK(h, EINVAL);
2613
2614         instr->type = INSTR_HDR_INVALIDATE;
2615         instr->valid.header_id = h->id;
2616         return 0;
2617 }
2618
2619 static inline void
2620 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2621 {
2622         struct thread *t = &p->threads[p->thread_id];
2623         struct instruction *ip = t->ip;
2624         uint32_t header_id = ip->valid.header_id;
2625
2626         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2627
2628         /* Headers. */
2629         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2630
2631         /* Thread. */
2632         thread_ip_inc(p);
2633 }
2634
2635 /*
2636  * mov.
2637  */
2638 static int
2639 instr_mov_translate(struct rte_swx_pipeline *p,
2640                     struct action *action,
2641                     char **tokens,
2642                     int n_tokens,
2643                     struct instruction *instr,
2644                     struct instruction_data *data __rte_unused)
2645 {
2646         char *dst = tokens[1], *src = tokens[2];
2647         struct field *fdst, *fsrc;
2648         uint32_t dst_struct_id, src_struct_id, src_val;
2649
2650         CHECK(n_tokens == 3, EINVAL);
2651
2652         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2653         CHECK(fdst, EINVAL);
2654
2655         /* MOV or MOV_S. */
2656         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2657         if (fsrc) {
2658                 instr->type = INSTR_MOV;
2659                 if ((dst[0] == 'h' && src[0] != 'h') ||
2660                     (dst[0] != 'h' && src[0] == 'h'))
2661                         instr->type = INSTR_MOV_S;
2662
2663                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2664                 instr->mov.dst.n_bits = fdst->n_bits;
2665                 instr->mov.dst.offset = fdst->offset / 8;
2666                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2667                 instr->mov.src.n_bits = fsrc->n_bits;
2668                 instr->mov.src.offset = fsrc->offset / 8;
2669                 return 0;
2670         }
2671
2672         /* MOV_I. */
2673         src_val = strtoul(src, &src, 0);
2674         CHECK(!src[0], EINVAL);
2675
2676         if (dst[0] == 'h')
2677                 src_val = htonl(src_val);
2678
2679         instr->type = INSTR_MOV_I;
2680         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2681         instr->mov.dst.n_bits = fdst->n_bits;
2682         instr->mov.dst.offset = fdst->offset / 8;
2683         instr->mov.src_val = (uint32_t)src_val;
2684         return 0;
2685 }
2686
2687 static inline void
2688 instr_mov_exec(struct rte_swx_pipeline *p)
2689 {
2690         struct thread *t = &p->threads[p->thread_id];
2691         struct instruction *ip = t->ip;
2692
2693         TRACE("[Thread %2u] mov\n",
2694               p->thread_id);
2695
2696         MOV(t, ip);
2697
2698         /* Thread. */
2699         thread_ip_inc(p);
2700 }
2701
2702 static inline void
2703 instr_mov_s_exec(struct rte_swx_pipeline *p)
2704 {
2705         struct thread *t = &p->threads[p->thread_id];
2706         struct instruction *ip = t->ip;
2707
2708         TRACE("[Thread %2u] mov (s)\n",
2709               p->thread_id);
2710
2711         MOV_S(t, ip);
2712
2713         /* Thread. */
2714         thread_ip_inc(p);
2715 }
2716
2717 static inline void
2718 instr_mov_i_exec(struct rte_swx_pipeline *p)
2719 {
2720         struct thread *t = &p->threads[p->thread_id];
2721         struct instruction *ip = t->ip;
2722
2723         TRACE("[Thread %2u] mov m.f %x\n",
2724               p->thread_id,
2725               ip->mov.src_val);
2726
2727         MOV_I(t, ip);
2728
2729         /* Thread. */
2730         thread_ip_inc(p);
2731 }
2732
2733 /*
2734  * dma.
2735  */
2736 static int
2737 instr_dma_translate(struct rte_swx_pipeline *p,
2738                     struct action *action,
2739                     char **tokens,
2740                     int n_tokens,
2741                     struct instruction *instr,
2742                     struct instruction_data *data __rte_unused)
2743 {
2744         char *dst = tokens[1];
2745         char *src = tokens[2];
2746         struct header *h;
2747         struct field *tf;
2748
2749         CHECK(action, EINVAL);
2750         CHECK(n_tokens == 3, EINVAL);
2751
2752         h = header_parse(p, dst);
2753         CHECK(h, EINVAL);
2754
2755         tf = action_field_parse(action, src);
2756         CHECK(tf, EINVAL);
2757
2758         instr->type = INSTR_DMA_HT;
2759         instr->dma.dst.header_id[0] = h->id;
2760         instr->dma.dst.struct_id[0] = h->struct_id;
2761         instr->dma.n_bytes[0] = h->st->n_bits / 8;
2762         instr->dma.src.offset[0] = tf->offset / 8;
2763
2764         return 0;
2765 }
2766
2767 static inline void
2768 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2769
2770 static inline void
2771 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2772 {
2773         struct thread *t = &p->threads[p->thread_id];
2774         struct instruction *ip = t->ip;
2775         uint8_t *action_data = t->structs[0];
2776         uint64_t valid_headers = t->valid_headers;
2777         uint32_t i;
2778
2779         for (i = 0; i < n_dma; i++) {
2780                 uint32_t header_id = ip->dma.dst.header_id[i];
2781                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2782                 uint32_t offset = ip->dma.src.offset[i];
2783                 uint32_t n_bytes = ip->dma.n_bytes[i];
2784
2785                 struct header_runtime *h = &t->headers[header_id];
2786                 uint8_t *h_ptr0 = h->ptr0;
2787                 uint8_t *h_ptr = t->structs[struct_id];
2788
2789                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2790                         h_ptr : h_ptr0;
2791                 void *src = &action_data[offset];
2792
2793                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2794
2795                 /* Headers. */
2796                 memcpy(dst, src, n_bytes);
2797                 t->structs[struct_id] = dst;
2798                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2799         }
2800
2801         t->valid_headers = valid_headers;
2802 }
2803
2804 static inline void
2805 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2806 {
2807         __instr_dma_ht_exec(p, 1);
2808
2809         /* Thread. */
2810         thread_ip_inc(p);
2811 }
2812
2813 static inline void
2814 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2815 {
2816         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2817               p->thread_id);
2818
2819         __instr_dma_ht_exec(p, 2);
2820
2821         /* Thread. */
2822         thread_ip_inc(p);
2823 }
2824
2825 static inline void
2826 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2827 {
2828         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2829               p->thread_id);
2830
2831         __instr_dma_ht_exec(p, 3);
2832
2833         /* Thread. */
2834         thread_ip_inc(p);
2835 }
2836
2837 static inline void
2838 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2839 {
2840         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2841               p->thread_id);
2842
2843         __instr_dma_ht_exec(p, 4);
2844
2845         /* Thread. */
2846         thread_ip_inc(p);
2847 }
2848
2849 static inline void
2850 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2851 {
2852         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2853               p->thread_id);
2854
2855         __instr_dma_ht_exec(p, 5);
2856
2857         /* Thread. */
2858         thread_ip_inc(p);
2859 }
2860
2861 static inline void
2862 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2863 {
2864         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2865               p->thread_id);
2866
2867         __instr_dma_ht_exec(p, 6);
2868
2869         /* Thread. */
2870         thread_ip_inc(p);
2871 }
2872
2873 static inline void
2874 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2875 {
2876         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2877               p->thread_id);
2878
2879         __instr_dma_ht_exec(p, 7);
2880
2881         /* Thread. */
2882         thread_ip_inc(p);
2883 }
2884
2885 static inline void
2886 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2887 {
2888         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2889               p->thread_id);
2890
2891         __instr_dma_ht_exec(p, 8);
2892
2893         /* Thread. */
2894         thread_ip_inc(p);
2895 }
2896
2897 /*
2898  * alu.
2899  */
2900 static int
2901 instr_alu_add_translate(struct rte_swx_pipeline *p,
2902                         struct action *action,
2903                         char **tokens,
2904                         int n_tokens,
2905                         struct instruction *instr,
2906                         struct instruction_data *data __rte_unused)
2907 {
2908         char *dst = tokens[1], *src = tokens[2];
2909         struct field *fdst, *fsrc;
2910         uint32_t dst_struct_id, src_struct_id, src_val;
2911
2912         CHECK(n_tokens == 3, EINVAL);
2913
2914         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2915         CHECK(fdst, EINVAL);
2916
2917         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2918         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2919         if (fsrc) {
2920                 instr->type = INSTR_ALU_ADD;
2921                 if (dst[0] == 'h' && src[0] == 'm')
2922                         instr->type = INSTR_ALU_ADD_HM;
2923                 if (dst[0] == 'm' && src[0] == 'h')
2924                         instr->type = INSTR_ALU_ADD_MH;
2925                 if (dst[0] == 'h' && src[0] == 'h')
2926                         instr->type = INSTR_ALU_ADD_HH;
2927
2928                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2929                 instr->alu.dst.n_bits = fdst->n_bits;
2930                 instr->alu.dst.offset = fdst->offset / 8;
2931                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2932                 instr->alu.src.n_bits = fsrc->n_bits;
2933                 instr->alu.src.offset = fsrc->offset / 8;
2934                 return 0;
2935         }
2936
2937         /* ADD_MI, ADD_HI. */
2938         src_val = strtoul(src, &src, 0);
2939         CHECK(!src[0], EINVAL);
2940
2941         instr->type = INSTR_ALU_ADD_MI;
2942         if (dst[0] == 'h')
2943                 instr->type = INSTR_ALU_ADD_HI;
2944
2945         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2946         instr->alu.dst.n_bits = fdst->n_bits;
2947         instr->alu.dst.offset = fdst->offset / 8;
2948         instr->alu.src_val = (uint32_t)src_val;
2949         return 0;
2950 }
2951
2952 static int
2953 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2954                         struct action *action,
2955                         char **tokens,
2956                         int n_tokens,
2957                         struct instruction *instr,
2958                         struct instruction_data *data __rte_unused)
2959 {
2960         char *dst = tokens[1], *src = tokens[2];
2961         struct field *fdst, *fsrc;
2962         uint32_t dst_struct_id, src_struct_id, src_val;
2963
2964         CHECK(n_tokens == 3, EINVAL);
2965
2966         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2967         CHECK(fdst, EINVAL);
2968
2969         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2970         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2971         if (fsrc) {
2972                 instr->type = INSTR_ALU_SUB;
2973                 if (dst[0] == 'h' && src[0] == 'm')
2974                         instr->type = INSTR_ALU_SUB_HM;
2975                 if (dst[0] == 'm' && src[0] == 'h')
2976                         instr->type = INSTR_ALU_SUB_MH;
2977                 if (dst[0] == 'h' && src[0] == 'h')
2978                         instr->type = INSTR_ALU_SUB_HH;
2979
2980                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2981                 instr->alu.dst.n_bits = fdst->n_bits;
2982                 instr->alu.dst.offset = fdst->offset / 8;
2983                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2984                 instr->alu.src.n_bits = fsrc->n_bits;
2985                 instr->alu.src.offset = fsrc->offset / 8;
2986                 return 0;
2987         }
2988
2989         /* SUB_MI, SUB_HI. */
2990         src_val = strtoul(src, &src, 0);
2991         CHECK(!src[0], EINVAL);
2992
2993         instr->type = INSTR_ALU_SUB_MI;
2994         if (dst[0] == 'h')
2995                 instr->type = INSTR_ALU_SUB_HI;
2996
2997         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2998         instr->alu.dst.n_bits = fdst->n_bits;
2999         instr->alu.dst.offset = fdst->offset / 8;
3000         instr->alu.src_val = (uint32_t)src_val;
3001         return 0;
3002 }
3003
3004 static int
3005 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3006                           struct action *action __rte_unused,
3007                           char **tokens,
3008                           int n_tokens,
3009                           struct instruction *instr,
3010                           struct instruction_data *data __rte_unused)
3011 {
3012         char *dst = tokens[1], *src = tokens[2];
3013         struct header *hdst, *hsrc;
3014         struct field *fdst, *fsrc;
3015
3016         CHECK(n_tokens == 3, EINVAL);
3017
3018         fdst = header_field_parse(p, dst, &hdst);
3019         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3020
3021         /* CKADD_FIELD. */
3022         fsrc = header_field_parse(p, src, &hsrc);
3023         if (fsrc) {
3024                 instr->type = INSTR_ALU_CKADD_FIELD;
3025                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3026                 instr->alu.dst.n_bits = fdst->n_bits;
3027                 instr->alu.dst.offset = fdst->offset / 8;
3028                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3029                 instr->alu.src.n_bits = fsrc->n_bits;
3030                 instr->alu.src.offset = fsrc->offset / 8;
3031                 return 0;
3032         }
3033
3034         /* CKADD_STRUCT, CKADD_STRUCT20. */
3035         hsrc = header_parse(p, src);
3036         CHECK(hsrc, EINVAL);
3037
3038         instr->type = INSTR_ALU_CKADD_STRUCT;
3039         if ((hsrc->st->n_bits / 8) == 20)
3040                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3041
3042         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3043         instr->alu.dst.n_bits = fdst->n_bits;
3044         instr->alu.dst.offset = fdst->offset / 8;
3045         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3046         instr->alu.src.n_bits = hsrc->st->n_bits;
3047         instr->alu.src.offset = 0; /* Unused. */
3048         return 0;
3049 }
3050
3051 static int
3052 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3053                           struct action *action __rte_unused,
3054                           char **tokens,
3055                           int n_tokens,
3056                           struct instruction *instr,
3057                           struct instruction_data *data __rte_unused)
3058 {
3059         char *dst = tokens[1], *src = tokens[2];
3060         struct header *hdst, *hsrc;
3061         struct field *fdst, *fsrc;
3062
3063         CHECK(n_tokens == 3, EINVAL);
3064
3065         fdst = header_field_parse(p, dst, &hdst);
3066         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3067
3068         fsrc = header_field_parse(p, src, &hsrc);
3069         CHECK(fsrc, EINVAL);
3070
3071         instr->type = INSTR_ALU_CKSUB_FIELD;
3072         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3073         instr->alu.dst.n_bits = fdst->n_bits;
3074         instr->alu.dst.offset = fdst->offset / 8;
3075         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3076         instr->alu.src.n_bits = fsrc->n_bits;
3077         instr->alu.src.offset = fsrc->offset / 8;
3078         return 0;
3079 }
3080
3081 static int
3082 instr_alu_and_translate(struct rte_swx_pipeline *p,
3083                         struct action *action,
3084                         char **tokens,
3085                         int n_tokens,
3086                         struct instruction *instr,
3087                         struct instruction_data *data __rte_unused)
3088 {
3089         char *dst = tokens[1], *src = tokens[2];
3090         struct field *fdst, *fsrc;
3091         uint32_t dst_struct_id, src_struct_id, src_val;
3092
3093         CHECK(n_tokens == 3, EINVAL);
3094
3095         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3096         CHECK(fdst, EINVAL);
3097
3098         /* AND or AND_S. */
3099         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3100         if (fsrc) {
3101                 instr->type = INSTR_ALU_AND;
3102                 if ((dst[0] == 'h' && src[0] != 'h') ||
3103                     (dst[0] != 'h' && src[0] == 'h'))
3104                         instr->type = INSTR_ALU_AND_S;
3105
3106                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3107                 instr->alu.dst.n_bits = fdst->n_bits;
3108                 instr->alu.dst.offset = fdst->offset / 8;
3109                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3110                 instr->alu.src.n_bits = fsrc->n_bits;
3111                 instr->alu.src.offset = fsrc->offset / 8;
3112                 return 0;
3113         }
3114
3115         /* AND_I. */
3116         src_val = strtoul(src, &src, 0);
3117         CHECK(!src[0], EINVAL);
3118
3119         if (dst[0] == 'h')
3120                 src_val = htonl(src_val);
3121
3122         instr->type = INSTR_ALU_AND_I;
3123         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3124         instr->alu.dst.n_bits = fdst->n_bits;
3125         instr->alu.dst.offset = fdst->offset / 8;
3126         instr->alu.src_val = (uint32_t)src_val;
3127         return 0;
3128 }
3129
3130 static inline void
3131 instr_alu_add_exec(struct rte_swx_pipeline *p)
3132 {
3133         struct thread *t = &p->threads[p->thread_id];
3134         struct instruction *ip = t->ip;
3135
3136         TRACE("[Thread %2u] add\n", p->thread_id);
3137
3138         /* Structs. */
3139         ALU(t, ip, +);
3140
3141         /* Thread. */
3142         thread_ip_inc(p);
3143 }
3144
3145 static inline void
3146 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3147 {
3148         struct thread *t = &p->threads[p->thread_id];
3149         struct instruction *ip = t->ip;
3150
3151         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3152
3153         /* Structs. */
3154         ALU_MH(t, ip, +);
3155
3156         /* Thread. */
3157         thread_ip_inc(p);
3158 }
3159
3160 static inline void
3161 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3162 {
3163         struct thread *t = &p->threads[p->thread_id];
3164         struct instruction *ip = t->ip;
3165
3166         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3167
3168         /* Structs. */
3169         ALU_HM(t, ip, +);
3170
3171         /* Thread. */
3172         thread_ip_inc(p);
3173 }
3174
3175 static inline void
3176 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3177 {
3178         struct thread *t = &p->threads[p->thread_id];
3179         struct instruction *ip = t->ip;
3180
3181         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3182
3183         /* Structs. */
3184         ALU_HH(t, ip, +);
3185
3186         /* Thread. */
3187         thread_ip_inc(p);
3188 }
3189
3190 static inline void
3191 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3192 {
3193         struct thread *t = &p->threads[p->thread_id];
3194         struct instruction *ip = t->ip;
3195
3196         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3197
3198         /* Structs. */
3199         ALU_MI(t, ip, +);
3200
3201         /* Thread. */
3202         thread_ip_inc(p);
3203 }
3204
3205 static inline void
3206 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3207 {
3208         struct thread *t = &p->threads[p->thread_id];
3209         struct instruction *ip = t->ip;
3210
3211         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3212
3213         /* Structs. */
3214         ALU_HI(t, ip, +);
3215
3216         /* Thread. */
3217         thread_ip_inc(p);
3218 }
3219
3220 static inline void
3221 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3222 {
3223         struct thread *t = &p->threads[p->thread_id];
3224         struct instruction *ip = t->ip;
3225
3226         TRACE("[Thread %2u] sub\n", p->thread_id);
3227
3228         /* Structs. */
3229         ALU(t, ip, -);
3230
3231         /* Thread. */
3232         thread_ip_inc(p);
3233 }
3234
3235 static inline void
3236 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3237 {
3238         struct thread *t = &p->threads[p->thread_id];
3239         struct instruction *ip = t->ip;
3240
3241         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3242
3243         /* Structs. */
3244         ALU_MH(t, ip, -);
3245
3246         /* Thread. */
3247         thread_ip_inc(p);
3248 }
3249
3250 static inline void
3251 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3252 {
3253         struct thread *t = &p->threads[p->thread_id];
3254         struct instruction *ip = t->ip;
3255
3256         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3257
3258         /* Structs. */
3259         ALU_HM(t, ip, -);
3260
3261         /* Thread. */
3262         thread_ip_inc(p);
3263 }
3264
3265 static inline void
3266 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3267 {
3268         struct thread *t = &p->threads[p->thread_id];
3269         struct instruction *ip = t->ip;
3270
3271         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3272
3273         /* Structs. */
3274         ALU_HH(t, ip, -);
3275
3276         /* Thread. */
3277         thread_ip_inc(p);
3278 }
3279
3280 static inline void
3281 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3282 {
3283         struct thread *t = &p->threads[p->thread_id];
3284         struct instruction *ip = t->ip;
3285
3286         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3287
3288         /* Structs. */
3289         ALU_MI(t, ip, -);
3290
3291         /* Thread. */
3292         thread_ip_inc(p);
3293 }
3294
3295 static inline void
3296 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3297 {
3298         struct thread *t = &p->threads[p->thread_id];
3299         struct instruction *ip = t->ip;
3300
3301         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3302
3303         /* Structs. */
3304         ALU_HI(t, ip, -);
3305
3306         /* Thread. */
3307         thread_ip_inc(p);
3308 }
3309
3310 static inline void
3311 instr_alu_and_exec(struct rte_swx_pipeline *p)
3312 {
3313         struct thread *t = &p->threads[p->thread_id];
3314         struct instruction *ip = t->ip;
3315
3316         TRACE("[Thread %2u] and\n", p->thread_id);
3317
3318         /* Structs. */
3319         ALU(t, ip, &);
3320
3321         /* Thread. */
3322         thread_ip_inc(p);
3323 }
3324
3325 static inline void
3326 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
3327 {
3328         struct thread *t = &p->threads[p->thread_id];
3329         struct instruction *ip = t->ip;
3330
3331         TRACE("[Thread %2u] and (s)\n", p->thread_id);
3332
3333         /* Structs. */
3334         ALU_S(t, ip, &);
3335
3336         /* Thread. */
3337         thread_ip_inc(p);
3338 }
3339
3340 static inline void
3341 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3342 {
3343         struct thread *t = &p->threads[p->thread_id];
3344         struct instruction *ip = t->ip;
3345
3346         TRACE("[Thread %2u] and (i)\n", p->thread_id);
3347
3348         /* Structs. */
3349         ALU_I(t, ip, &);
3350
3351         /* Thread. */
3352         thread_ip_inc(p);
3353 }
3354
3355 static inline void
3356 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3357 {
3358         struct thread *t = &p->threads[p->thread_id];
3359         struct instruction *ip = t->ip;
3360         uint8_t *dst_struct, *src_struct;
3361         uint16_t *dst16_ptr, dst;
3362         uint64_t *src64_ptr, src64, src64_mask, src;
3363         uint64_t r;
3364
3365         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3366
3367         /* Structs. */
3368         dst_struct = t->structs[ip->alu.dst.struct_id];
3369         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3370         dst = *dst16_ptr;
3371
3372         src_struct = t->structs[ip->alu.src.struct_id];
3373         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3374         src64 = *src64_ptr;
3375         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3376         src = src64 & src64_mask;
3377
3378         r = dst;
3379         r = ~r & 0xFFFF;
3380
3381         /* The first input (r) is a 16-bit number. The second and the third
3382          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3383          * three numbers (output r) is a 34-bit number.
3384          */
3385         r += (src >> 32) + (src & 0xFFFFFFFF);
3386
3387         /* The first input is a 16-bit number. The second input is an 18-bit
3388          * number. In the worst case scenario, the sum of the two numbers is a
3389          * 19-bit number.
3390          */
3391         r = (r & 0xFFFF) + (r >> 16);
3392
3393         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3394          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3395          */
3396         r = (r & 0xFFFF) + (r >> 16);
3397
3398         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3399          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3400          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3401          * therefore the output r is always a 16-bit number.
3402          */
3403         r = (r & 0xFFFF) + (r >> 16);
3404
3405         r = ~r & 0xFFFF;
3406         r = r ? r : 0xFFFF;
3407
3408         *dst16_ptr = (uint16_t)r;
3409
3410         /* Thread. */
3411         thread_ip_inc(p);
3412 }
3413
3414 static inline void
3415 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3416 {
3417         struct thread *t = &p->threads[p->thread_id];
3418         struct instruction *ip = t->ip;
3419         uint8_t *dst_struct, *src_struct;
3420         uint16_t *dst16_ptr, dst;
3421         uint64_t *src64_ptr, src64, src64_mask, src;
3422         uint64_t r;
3423
3424         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3425
3426         /* Structs. */
3427         dst_struct = t->structs[ip->alu.dst.struct_id];
3428         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3429         dst = *dst16_ptr;
3430
3431         src_struct = t->structs[ip->alu.src.struct_id];
3432         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3433         src64 = *src64_ptr;
3434         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3435         src = src64 & src64_mask;
3436
3437         r = dst;
3438         r = ~r & 0xFFFF;
3439
3440         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3441          * the following sequence of operations in 2's complement arithmetic:
3442          *    a '- b = (a - b) % 0xFFFF.
3443          *
3444          * In order to prevent an underflow for the below subtraction, in which
3445          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3446          * minuend), we first add a multiple of the 0xFFFF modulus to the
3447          * minuend. The number we add to the minuend needs to be a 34-bit number
3448          * or higher, so for readability reasons we picked the 36-bit multiple.
3449          * We are effectively turning the 16-bit minuend into a 36-bit number:
3450          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3451          */
3452         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3453
3454         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3455          * result (the output r) is a 36-bit number.
3456          */
3457         r -= (src >> 32) + (src & 0xFFFFFFFF);
3458
3459         /* The first input is a 16-bit number. The second input is a 20-bit
3460          * number. Their sum is a 21-bit number.
3461          */
3462         r = (r & 0xFFFF) + (r >> 16);
3463
3464         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3465          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3466          */
3467         r = (r & 0xFFFF) + (r >> 16);
3468
3469         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3470          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3471          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3472          * generated, therefore the output r is always a 16-bit number.
3473          */
3474         r = (r & 0xFFFF) + (r >> 16);
3475
3476         r = ~r & 0xFFFF;
3477         r = r ? r : 0xFFFF;
3478
3479         *dst16_ptr = (uint16_t)r;
3480
3481         /* Thread. */
3482         thread_ip_inc(p);
3483 }
3484
3485 static inline void
3486 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3487 {
3488         struct thread *t = &p->threads[p->thread_id];
3489         struct instruction *ip = t->ip;
3490         uint8_t *dst_struct, *src_struct;
3491         uint16_t *dst16_ptr;
3492         uint32_t *src32_ptr;
3493         uint64_t r0, r1;
3494
3495         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3496
3497         /* Structs. */
3498         dst_struct = t->structs[ip->alu.dst.struct_id];
3499         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3500
3501         src_struct = t->structs[ip->alu.src.struct_id];
3502         src32_ptr = (uint32_t *)&src_struct[0];
3503
3504         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
3505         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3506         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
3507         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3508         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3509
3510         /* The first input is a 16-bit number. The second input is a 19-bit
3511          * number. Their sum is a 20-bit number.
3512          */
3513         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3514
3515         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3516          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3517          */
3518         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3519
3520         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3521          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3522          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3523          * generated, therefore the output r is always a 16-bit number.
3524          */
3525         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3526
3527         r0 = ~r0 & 0xFFFF;
3528         r0 = r0 ? r0 : 0xFFFF;
3529
3530         *dst16_ptr = (uint16_t)r0;
3531
3532         /* Thread. */
3533         thread_ip_inc(p);
3534 }
3535
3536 static inline void
3537 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3538 {
3539         struct thread *t = &p->threads[p->thread_id];
3540         struct instruction *ip = t->ip;
3541         uint8_t *dst_struct, *src_struct;
3542         uint16_t *dst16_ptr;
3543         uint32_t *src32_ptr;
3544         uint64_t r = 0;
3545         uint32_t i;
3546
3547         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3548
3549         /* Structs. */
3550         dst_struct = t->structs[ip->alu.dst.struct_id];
3551         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3552
3553         src_struct = t->structs[ip->alu.src.struct_id];
3554         src32_ptr = (uint32_t *)&src_struct[0];
3555
3556         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
3557          * Therefore, in the worst case scenario, a 35-bit number is added to a
3558          * 16-bit number (the input r), so the output r is 36-bit number.
3559          */
3560         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
3561                 r += *src32_ptr;
3562
3563         /* The first input is a 16-bit number. The second input is a 20-bit
3564          * number. Their sum is a 21-bit number.
3565          */
3566         r = (r & 0xFFFF) + (r >> 16);
3567
3568         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3569          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
3570          */
3571         r = (r & 0xFFFF) + (r >> 16);
3572
3573         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3574          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3575          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3576          * generated, therefore the output r is always a 16-bit number.
3577          */
3578         r = (r & 0xFFFF) + (r >> 16);
3579
3580         r = ~r & 0xFFFF;
3581         r = r ? r : 0xFFFF;
3582
3583         *dst16_ptr = (uint16_t)r;
3584
3585         /* Thread. */
3586         thread_ip_inc(p);
3587 }
3588
3589 #define RTE_SWX_INSTRUCTION_TOKENS_MAX 16
3590
3591 static int
3592 instr_translate(struct rte_swx_pipeline *p,
3593                 struct action *action,
3594                 char *string,
3595                 struct instruction *instr,
3596                 struct instruction_data *data)
3597 {
3598         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
3599         int n_tokens = 0, tpos = 0;
3600
3601         /* Parse the instruction string into tokens. */
3602         for ( ; ; ) {
3603                 char *token;
3604
3605                 token = strtok_r(string, " \t\v", &string);
3606                 if (!token)
3607                         break;
3608
3609                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
3610
3611                 tokens[n_tokens] = token;
3612                 n_tokens++;
3613         }
3614
3615         CHECK(n_tokens, EINVAL);
3616
3617         /* Handle the optional instruction label. */
3618         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
3619                 strcpy(data->label, tokens[0]);
3620
3621                 tpos += 2;
3622                 CHECK(n_tokens - tpos, EINVAL);
3623         }
3624
3625         /* Identify the instruction type. */
3626         if (!strcmp(tokens[tpos], "rx"))
3627                 return instr_rx_translate(p,
3628                                           action,
3629                                           &tokens[tpos],
3630                                           n_tokens - tpos,
3631                                           instr,
3632                                           data);
3633
3634         if (!strcmp(tokens[tpos], "tx"))
3635                 return instr_tx_translate(p,
3636                                           action,
3637                                           &tokens[tpos],
3638                                           n_tokens - tpos,
3639                                           instr,
3640                                           data);
3641
3642         if (!strcmp(tokens[tpos], "extract"))
3643                 return instr_hdr_extract_translate(p,
3644                                                    action,
3645                                                    &tokens[tpos],
3646                                                    n_tokens - tpos,
3647                                                    instr,
3648                                                    data);
3649
3650         if (!strcmp(tokens[tpos], "emit"))
3651                 return instr_hdr_emit_translate(p,
3652                                                 action,
3653                                                 &tokens[tpos],
3654                                                 n_tokens - tpos,
3655                                                 instr,
3656                                                 data);
3657
3658         if (!strcmp(tokens[tpos], "validate"))
3659                 return instr_hdr_validate_translate(p,
3660                                                     action,
3661                                                     &tokens[tpos],
3662                                                     n_tokens - tpos,
3663                                                     instr,
3664                                                     data);
3665
3666         if (!strcmp(tokens[tpos], "invalidate"))
3667                 return instr_hdr_invalidate_translate(p,
3668                                                       action,
3669                                                       &tokens[tpos],
3670                                                       n_tokens - tpos,
3671                                                       instr,
3672                                                       data);
3673
3674         if (!strcmp(tokens[tpos], "mov"))
3675                 return instr_mov_translate(p,
3676                                            action,
3677                                            &tokens[tpos],
3678                                            n_tokens - tpos,
3679                                            instr,
3680                                            data);
3681
3682         if (!strcmp(tokens[tpos], "dma"))
3683                 return instr_dma_translate(p,
3684                                            action,
3685                                            &tokens[tpos],
3686                                            n_tokens - tpos,
3687                                            instr,
3688                                            data);
3689
3690         if (!strcmp(tokens[tpos], "add"))
3691                 return instr_alu_add_translate(p,
3692                                                action,
3693                                                &tokens[tpos],
3694                                                n_tokens - tpos,
3695                                                instr,
3696                                                data);
3697
3698         if (!strcmp(tokens[tpos], "sub"))
3699                 return instr_alu_sub_translate(p,
3700                                                action,
3701                                                &tokens[tpos],
3702                                                n_tokens - tpos,
3703                                                instr,
3704                                                data);
3705
3706         if (!strcmp(tokens[tpos], "ckadd"))
3707                 return instr_alu_ckadd_translate(p,
3708                                                  action,
3709                                                  &tokens[tpos],
3710                                                  n_tokens - tpos,
3711                                                  instr,
3712                                                  data);
3713
3714         if (!strcmp(tokens[tpos], "cksub"))
3715                 return instr_alu_cksub_translate(p,
3716                                                  action,
3717                                                  &tokens[tpos],
3718                                                  n_tokens - tpos,
3719                                                  instr,
3720                                                  data);
3721
3722         if (!strcmp(tokens[tpos], "and"))
3723                 return instr_alu_and_translate(p,
3724                                                action,
3725                                                &tokens[tpos],
3726                                                n_tokens - tpos,
3727                                                instr,
3728                                                data);
3729
3730         CHECK(0, EINVAL);
3731 }
3732
3733 static uint32_t
3734 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
3735 {
3736         uint32_t count = 0, i;
3737
3738         if (!label[0])
3739                 return 0;
3740
3741         for (i = 0; i < n; i++)
3742                 if (!strcmp(label, data[i].jmp_label))
3743                         count++;
3744
3745         return count;
3746 }
3747
3748 static int
3749 instr_label_check(struct instruction_data *instruction_data,
3750                   uint32_t n_instructions)
3751 {
3752         uint32_t i;
3753
3754         /* Check that all instruction labels are unique. */
3755         for (i = 0; i < n_instructions; i++) {
3756                 struct instruction_data *data = &instruction_data[i];
3757                 char *label = data->label;
3758                 uint32_t j;
3759
3760                 if (!label[0])
3761                         continue;
3762
3763                 for (j = i + 1; j < n_instructions; j++)
3764                         CHECK(strcmp(label, data[j].label), EINVAL);
3765         }
3766
3767         /* Get users for each instruction label. */
3768         for (i = 0; i < n_instructions; i++) {
3769                 struct instruction_data *data = &instruction_data[i];
3770                 char *label = data->label;
3771
3772                 data->n_users = label_is_used(instruction_data,
3773                                               n_instructions,
3774                                               label);
3775         }
3776
3777         return 0;
3778 }
3779
3780 static int
3781 instruction_config(struct rte_swx_pipeline *p,
3782                    struct action *a,
3783                    const char **instructions,
3784                    uint32_t n_instructions)
3785 {
3786         struct instruction *instr = NULL;
3787         struct instruction_data *data = NULL;
3788         char *string = NULL;
3789         int err = 0;
3790         uint32_t i;
3791
3792         CHECK(n_instructions, EINVAL);
3793         CHECK(instructions, EINVAL);
3794         for (i = 0; i < n_instructions; i++)
3795                 CHECK(instructions[i], EINVAL);
3796
3797         /* Memory allocation. */
3798         instr = calloc(n_instructions, sizeof(struct instruction));
3799         if (!instr) {
3800                 err = ENOMEM;
3801                 goto error;
3802         }
3803
3804         data = calloc(n_instructions, sizeof(struct instruction_data));
3805         if (!data) {
3806                 err = ENOMEM;
3807                 goto error;
3808         }
3809
3810         for (i = 0; i < n_instructions; i++) {
3811                 string = strdup(instructions[i]);
3812                 if (!string) {
3813                         err = ENOMEM;
3814                         goto error;
3815                 }
3816
3817                 err = instr_translate(p, a, string, &instr[i], &data[i]);
3818                 if (err)
3819                         goto error;
3820
3821                 free(string);
3822         }
3823
3824         err = instr_label_check(data, n_instructions);
3825         if (err)
3826                 goto error;
3827
3828         free(data);
3829
3830         if (a) {
3831                 a->instructions = instr;
3832                 a->n_instructions = n_instructions;
3833         } else {
3834                 p->instructions = instr;
3835                 p->n_instructions = n_instructions;
3836         }
3837
3838         return 0;
3839
3840 error:
3841         free(string);
3842         free(data);
3843         free(instr);
3844         return err;
3845 }
3846
3847 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
3848
3849 static instr_exec_t instruction_table[] = {
3850         [INSTR_RX] = instr_rx_exec,
3851         [INSTR_TX] = instr_tx_exec,
3852
3853         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
3854         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
3855         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
3856         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
3857         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
3858         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
3859         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
3860         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
3861
3862         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
3863         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
3864         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
3865         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
3866         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
3867         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
3868         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
3869         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
3870         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
3871
3872         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
3873         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
3874
3875         [INSTR_MOV] = instr_mov_exec,
3876         [INSTR_MOV_S] = instr_mov_s_exec,
3877         [INSTR_MOV_I] = instr_mov_i_exec,
3878
3879         [INSTR_DMA_HT] = instr_dma_ht_exec,
3880         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
3881         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
3882         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
3883         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
3884         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
3885         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
3886         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
3887
3888         [INSTR_ALU_ADD] = instr_alu_add_exec,
3889         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
3890         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
3891         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
3892         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
3893         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
3894
3895         [INSTR_ALU_SUB] = instr_alu_sub_exec,
3896         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
3897         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
3898         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
3899         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
3900         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
3901
3902         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
3903         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
3904         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
3905         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
3906
3907         [INSTR_ALU_AND] = instr_alu_and_exec,
3908         [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
3909         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
3910 };
3911
3912 static inline void
3913 instr_exec(struct rte_swx_pipeline *p)
3914 {
3915         struct thread *t = &p->threads[p->thread_id];
3916         struct instruction *ip = t->ip;
3917         instr_exec_t instr = instruction_table[ip->type];
3918
3919         instr(p);
3920 }
3921
3922 /*
3923  * Action.
3924  */
3925 static struct action *
3926 action_find(struct rte_swx_pipeline *p, const char *name)
3927 {
3928         struct action *elem;
3929
3930         if (!name)
3931                 return NULL;
3932
3933         TAILQ_FOREACH(elem, &p->actions, node)
3934                 if (strcmp(elem->name, name) == 0)
3935                         return elem;
3936
3937         return NULL;
3938 }
3939
3940 static struct field *
3941 action_field_find(struct action *a, const char *name)
3942 {
3943         return a->st ? struct_type_field_find(a->st, name) : NULL;
3944 }
3945
3946 static struct field *
3947 action_field_parse(struct action *action, const char *name)
3948 {
3949         if (name[0] != 't' || name[1] != '.')
3950                 return NULL;
3951
3952         return action_field_find(action, &name[2]);
3953 }
3954
3955 int
3956 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
3957                                const char *name,
3958                                const char *args_struct_type_name,
3959                                const char **instructions,
3960                                uint32_t n_instructions)
3961 {
3962         struct struct_type *args_struct_type;
3963         struct action *a;
3964         int err;
3965
3966         CHECK(p, EINVAL);
3967
3968         CHECK_NAME(name, EINVAL);
3969         CHECK(!action_find(p, name), EEXIST);
3970
3971         if (args_struct_type_name) {
3972                 CHECK_NAME(args_struct_type_name, EINVAL);
3973                 args_struct_type = struct_type_find(p, args_struct_type_name);
3974                 CHECK(args_struct_type, EINVAL);
3975         } else {
3976                 args_struct_type = NULL;
3977         }
3978
3979         /* Node allocation. */
3980         a = calloc(1, sizeof(struct action));
3981         CHECK(a, ENOMEM);
3982
3983         /* Node initialization. */
3984         strcpy(a->name, name);
3985         a->st = args_struct_type;
3986         a->id = p->n_actions;
3987
3988         /* Instruction translation. */
3989         err = instruction_config(p, a, instructions, n_instructions);
3990         if (err) {
3991                 free(a);
3992                 return err;
3993         }
3994
3995         /* Node add to tailq. */
3996         TAILQ_INSERT_TAIL(&p->actions, a, node);
3997         p->n_actions++;
3998
3999         return 0;
4000 }
4001
4002 static int
4003 action_build(struct rte_swx_pipeline *p)
4004 {
4005         struct action *action;
4006
4007         p->action_instructions = calloc(p->n_actions,
4008                                         sizeof(struct instruction *));
4009         CHECK(p->action_instructions, ENOMEM);
4010
4011         TAILQ_FOREACH(action, &p->actions, node)
4012                 p->action_instructions[action->id] = action->instructions;
4013
4014         return 0;
4015 }
4016
4017 static void
4018 action_build_free(struct rte_swx_pipeline *p)
4019 {
4020         free(p->action_instructions);
4021         p->action_instructions = NULL;
4022 }
4023
4024 static void
4025 action_free(struct rte_swx_pipeline *p)
4026 {
4027         action_build_free(p);
4028
4029         for ( ; ; ) {
4030                 struct action *action;
4031
4032                 action = TAILQ_FIRST(&p->actions);
4033                 if (!action)
4034                         break;
4035
4036                 TAILQ_REMOVE(&p->actions, action, node);
4037                 free(action->instructions);
4038                 free(action);
4039         }
4040 }
4041
4042 /*
4043  * Table.
4044  */
4045 static struct table_type *
4046 table_type_find(struct rte_swx_pipeline *p, const char *name)
4047 {
4048         struct table_type *elem;
4049
4050         TAILQ_FOREACH(elem, &p->table_types, node)
4051                 if (strcmp(elem->name, name) == 0)
4052                         return elem;
4053
4054         return NULL;
4055 }
4056
4057 static struct table_type *
4058 table_type_resolve(struct rte_swx_pipeline *p,
4059                    const char *recommended_type_name,
4060                    enum rte_swx_table_match_type match_type)
4061 {
4062         struct table_type *elem;
4063
4064         /* Only consider the recommended type if the match type is correct. */
4065         if (recommended_type_name)
4066                 TAILQ_FOREACH(elem, &p->table_types, node)
4067                         if (!strcmp(elem->name, recommended_type_name) &&
4068                             (elem->match_type == match_type))
4069                                 return elem;
4070
4071         /* Ignore the recommended type and get the first element with this match
4072          * type.
4073          */
4074         TAILQ_FOREACH(elem, &p->table_types, node)
4075                 if (elem->match_type == match_type)
4076                         return elem;
4077
4078         return NULL;
4079 }
4080
4081 static struct table *
4082 table_find(struct rte_swx_pipeline *p, const char *name)
4083 {
4084         struct table *elem;
4085
4086         TAILQ_FOREACH(elem, &p->tables, node)
4087                 if (strcmp(elem->name, name) == 0)
4088                         return elem;
4089
4090         return NULL;
4091 }
4092
4093 static struct table *
4094 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
4095 {
4096         struct table *table = NULL;
4097
4098         TAILQ_FOREACH(table, &p->tables, node)
4099                 if (table->id == id)
4100                         return table;
4101
4102         return NULL;
4103 }
4104
4105 int
4106 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
4107                                      const char *name,
4108                                      enum rte_swx_table_match_type match_type,
4109                                      struct rte_swx_table_ops *ops)
4110 {
4111         struct table_type *elem;
4112
4113         CHECK(p, EINVAL);
4114
4115         CHECK_NAME(name, EINVAL);
4116         CHECK(!table_type_find(p, name), EEXIST);
4117
4118         CHECK(ops, EINVAL);
4119         CHECK(ops->create, EINVAL);
4120         CHECK(ops->lkp, EINVAL);
4121         CHECK(ops->free, EINVAL);
4122
4123         /* Node allocation. */
4124         elem = calloc(1, sizeof(struct table_type));
4125         CHECK(elem, ENOMEM);
4126
4127         /* Node initialization. */
4128         strcpy(elem->name, name);
4129         elem->match_type = match_type;
4130         memcpy(&elem->ops, ops, sizeof(*ops));
4131
4132         /* Node add to tailq. */
4133         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
4134
4135         return 0;
4136 }
4137
4138 static enum rte_swx_table_match_type
4139 table_match_type_resolve(struct rte_swx_match_field_params *fields,
4140                          uint32_t n_fields)
4141 {
4142         uint32_t i;
4143
4144         for (i = 0; i < n_fields; i++)
4145                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
4146                         break;
4147
4148         if (i == n_fields)
4149                 return RTE_SWX_TABLE_MATCH_EXACT;
4150
4151         if ((i == n_fields - 1) &&
4152             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
4153                 return RTE_SWX_TABLE_MATCH_LPM;
4154
4155         return RTE_SWX_TABLE_MATCH_WILDCARD;
4156 }
4157
4158 int
4159 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
4160                               const char *name,
4161                               struct rte_swx_pipeline_table_params *params,
4162                               const char *recommended_table_type_name,
4163                               const char *args,
4164                               uint32_t size)
4165 {
4166         struct table_type *type;
4167         struct table *t;
4168         struct action *default_action;
4169         struct header *header = NULL;
4170         int is_header = 0;
4171         uint32_t offset_prev = 0, action_data_size_max = 0, i;
4172
4173         CHECK(p, EINVAL);
4174
4175         CHECK_NAME(name, EINVAL);
4176         CHECK(!table_find(p, name), EEXIST);
4177
4178         CHECK(params, EINVAL);
4179
4180         /* Match checks. */
4181         CHECK(!params->n_fields || params->fields, EINVAL);
4182         for (i = 0; i < params->n_fields; i++) {
4183                 struct rte_swx_match_field_params *field = &params->fields[i];
4184                 struct header *h;
4185                 struct field *hf, *mf;
4186                 uint32_t offset;
4187
4188                 CHECK_NAME(field->name, EINVAL);
4189
4190                 hf = header_field_parse(p, field->name, &h);
4191                 mf = metadata_field_parse(p, field->name);
4192                 CHECK(hf || mf, EINVAL);
4193
4194                 offset = hf ? hf->offset : mf->offset;
4195
4196                 if (i == 0) {
4197                         is_header = hf ? 1 : 0;
4198                         header = hf ? h : NULL;
4199                         offset_prev = offset;
4200
4201                         continue;
4202                 }
4203
4204                 CHECK((is_header && hf && (h->id == header->id)) ||
4205                       (!is_header && mf), EINVAL);
4206
4207                 CHECK(offset > offset_prev, EINVAL);
4208                 offset_prev = offset;
4209         }
4210
4211         /* Action checks. */
4212         CHECK(params->n_actions, EINVAL);
4213         CHECK(params->action_names, EINVAL);
4214         for (i = 0; i < params->n_actions; i++) {
4215                 const char *action_name = params->action_names[i];
4216                 struct action *a;
4217                 uint32_t action_data_size;
4218
4219                 CHECK(action_name, EINVAL);
4220
4221                 a = action_find(p, action_name);
4222                 CHECK(a, EINVAL);
4223
4224                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
4225                 if (action_data_size > action_data_size_max)
4226                         action_data_size_max = action_data_size;
4227         }
4228
4229         CHECK(params->default_action_name, EINVAL);
4230         for (i = 0; i < p->n_actions; i++)
4231                 if (!strcmp(params->action_names[i],
4232                             params->default_action_name))
4233                         break;
4234         CHECK(i < params->n_actions, EINVAL);
4235         default_action = action_find(p, params->default_action_name);
4236         CHECK((default_action->st && params->default_action_data) ||
4237               !params->default_action_data, EINVAL);
4238
4239         /* Table type checks. */
4240         if (params->n_fields) {
4241                 enum rte_swx_table_match_type match_type;
4242
4243                 match_type = table_match_type_resolve(params->fields,
4244                                                       params->n_fields);
4245                 type = table_type_resolve(p,
4246                                           recommended_table_type_name,
4247                                           match_type);
4248                 CHECK(type, EINVAL);
4249         } else {
4250                 type = NULL;
4251         }
4252
4253         /* Memory allocation. */
4254         t = calloc(1, sizeof(struct table));
4255         CHECK(t, ENOMEM);
4256
4257         t->fields = calloc(params->n_fields, sizeof(struct match_field));
4258         if (!t->fields) {
4259                 free(t);
4260                 CHECK(0, ENOMEM);
4261         }
4262
4263         t->actions = calloc(params->n_actions, sizeof(struct action *));
4264         if (!t->actions) {
4265                 free(t->fields);
4266                 free(t);
4267                 CHECK(0, ENOMEM);
4268         }
4269
4270         if (action_data_size_max) {
4271                 t->default_action_data = calloc(1, action_data_size_max);
4272                 if (!t->default_action_data) {
4273                         free(t->actions);
4274                         free(t->fields);
4275                         free(t);
4276                         CHECK(0, ENOMEM);
4277                 }
4278         }
4279
4280         /* Node initialization. */
4281         strcpy(t->name, name);
4282         if (args && args[0])
4283                 strcpy(t->args, args);
4284         t->type = type;
4285
4286         for (i = 0; i < params->n_fields; i++) {
4287                 struct rte_swx_match_field_params *field = &params->fields[i];
4288                 struct match_field *f = &t->fields[i];
4289
4290                 f->match_type = field->match_type;
4291                 f->field = is_header ?
4292                         header_field_parse(p, field->name, NULL) :
4293                         metadata_field_parse(p, field->name);
4294         }
4295         t->n_fields = params->n_fields;
4296         t->is_header = is_header;
4297         t->header = header;
4298
4299         for (i = 0; i < params->n_actions; i++)
4300                 t->actions[i] = action_find(p, params->action_names[i]);
4301         t->default_action = default_action;
4302         if (default_action->st)
4303                 memcpy(t->default_action_data,
4304                        params->default_action_data,
4305                        default_action->st->n_bits / 8);
4306         t->n_actions = params->n_actions;
4307         t->default_action_is_const = params->default_action_is_const;
4308         t->action_data_size_max = action_data_size_max;
4309
4310         t->size = size;
4311         t->id = p->n_tables;
4312
4313         /* Node add to tailq. */
4314         TAILQ_INSERT_TAIL(&p->tables, t, node);
4315         p->n_tables++;
4316
4317         return 0;
4318 }
4319
4320 static struct rte_swx_table_params *
4321 table_params_get(struct table *table)
4322 {
4323         struct rte_swx_table_params *params;
4324         struct field *first, *last;
4325         uint8_t *key_mask;
4326         uint32_t key_size, key_offset, action_data_size, i;
4327
4328         /* Memory allocation. */
4329         params = calloc(1, sizeof(struct rte_swx_table_params));
4330         if (!params)
4331                 return NULL;
4332
4333         /* Key offset and size. */
4334         first = table->fields[0].field;
4335         last = table->fields[table->n_fields - 1].field;
4336         key_offset = first->offset / 8;
4337         key_size = (last->offset + last->n_bits - first->offset) / 8;
4338
4339         /* Memory allocation. */
4340         key_mask = calloc(1, key_size);
4341         if (!key_mask) {
4342                 free(params);
4343                 return NULL;
4344         }
4345
4346         /* Key mask. */
4347         for (i = 0; i < table->n_fields; i++) {
4348                 struct field *f = table->fields[i].field;
4349                 uint32_t start = (f->offset - first->offset) / 8;
4350                 size_t size = f->n_bits / 8;
4351
4352                 memset(&key_mask[start], 0xFF, size);
4353         }
4354
4355         /* Action data size. */
4356         action_data_size = 0;
4357         for (i = 0; i < table->n_actions; i++) {
4358                 struct action *action = table->actions[i];
4359                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
4360
4361                 if (ads > action_data_size)
4362                         action_data_size = ads;
4363         }
4364
4365         /* Fill in. */
4366         params->match_type = table->type->match_type;
4367         params->key_size = key_size;
4368         params->key_offset = key_offset;
4369         params->key_mask0 = key_mask;
4370         params->action_data_size = action_data_size;
4371         params->n_keys_max = table->size;
4372
4373         return params;
4374 }
4375
4376 static void
4377 table_params_free(struct rte_swx_table_params *params)
4378 {
4379         if (!params)
4380                 return;
4381
4382         free(params->key_mask0);
4383         free(params);
4384 }
4385
4386 static int
4387 table_state_build(struct rte_swx_pipeline *p)
4388 {
4389         struct table *table;
4390
4391         p->table_state = calloc(p->n_tables,
4392                                 sizeof(struct rte_swx_table_state));
4393         CHECK(p->table_state, ENOMEM);
4394
4395         TAILQ_FOREACH(table, &p->tables, node) {
4396                 struct rte_swx_table_state *ts = &p->table_state[table->id];
4397
4398                 if (table->type) {
4399                         struct rte_swx_table_params *params;
4400
4401                         /* ts->obj. */
4402                         params = table_params_get(table);
4403                         CHECK(params, ENOMEM);
4404
4405                         ts->obj = table->type->ops.create(params,
4406                                 NULL,
4407                                 table->args,
4408                                 p->numa_node);
4409
4410                         table_params_free(params);
4411                         CHECK(ts->obj, ENODEV);
4412                 }
4413
4414                 /* ts->default_action_data. */
4415                 if (table->action_data_size_max) {
4416                         ts->default_action_data =
4417                                 malloc(table->action_data_size_max);
4418                         CHECK(ts->default_action_data, ENOMEM);
4419
4420                         memcpy(ts->default_action_data,
4421                                table->default_action_data,
4422                                table->action_data_size_max);
4423                 }
4424
4425                 /* ts->default_action_id. */
4426                 ts->default_action_id = table->default_action->id;
4427         }
4428
4429         return 0;
4430 }
4431
4432 static void
4433 table_state_build_free(struct rte_swx_pipeline *p)
4434 {
4435         uint32_t i;
4436
4437         if (!p->table_state)
4438                 return;
4439
4440         for (i = 0; i < p->n_tables; i++) {
4441                 struct rte_swx_table_state *ts = &p->table_state[i];
4442                 struct table *table = table_find_by_id(p, i);
4443
4444                 /* ts->obj. */
4445                 if (table->type && ts->obj)
4446                         table->type->ops.free(ts->obj);
4447
4448                 /* ts->default_action_data. */
4449                 free(ts->default_action_data);
4450         }
4451
4452         free(p->table_state);
4453         p->table_state = NULL;
4454 }
4455
4456 static void
4457 table_state_free(struct rte_swx_pipeline *p)
4458 {
4459         table_state_build_free(p);
4460 }
4461
4462 static int
4463 table_stub_lkp(void *table __rte_unused,
4464                void *mailbox __rte_unused,
4465                uint8_t **key __rte_unused,
4466                uint64_t *action_id __rte_unused,
4467                uint8_t **action_data __rte_unused,
4468                int *hit)
4469 {
4470         *hit = 0;
4471         return 1; /* DONE. */
4472 }
4473
4474 static int
4475 table_build(struct rte_swx_pipeline *p)
4476 {
4477         uint32_t i;
4478
4479         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
4480                 struct thread *t = &p->threads[i];
4481                 struct table *table;
4482
4483                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
4484                 CHECK(t->tables, ENOMEM);
4485
4486                 TAILQ_FOREACH(table, &p->tables, node) {
4487                         struct table_runtime *r = &t->tables[table->id];
4488
4489                         if (table->type) {
4490                                 uint64_t size;
4491
4492                                 size = table->type->ops.mailbox_size_get();
4493
4494                                 /* r->func. */
4495                                 r->func = table->type->ops.lkp;
4496
4497                                 /* r->mailbox. */
4498                                 if (size) {
4499                                         r->mailbox = calloc(1, size);
4500                                         CHECK(r->mailbox, ENOMEM);
4501                                 }
4502
4503                                 /* r->key. */
4504                                 r->key = table->is_header ?
4505                                         &t->structs[table->header->struct_id] :
4506                                         &t->structs[p->metadata_struct_id];
4507                         } else {
4508                                 r->func = table_stub_lkp;
4509                         }
4510                 }
4511         }
4512
4513         return 0;
4514 }
4515
4516 static void
4517 table_build_free(struct rte_swx_pipeline *p)
4518 {
4519         uint32_t i;
4520
4521         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
4522                 struct thread *t = &p->threads[i];
4523                 uint32_t j;
4524
4525                 if (!t->tables)
4526                         continue;
4527
4528                 for (j = 0; j < p->n_tables; j++) {
4529                         struct table_runtime *r = &t->tables[j];
4530
4531                         free(r->mailbox);
4532                 }
4533
4534                 free(t->tables);
4535                 t->tables = NULL;
4536         }
4537 }
4538
4539 static void
4540 table_free(struct rte_swx_pipeline *p)
4541 {
4542         table_build_free(p);
4543
4544         /* Tables. */
4545         for ( ; ; ) {
4546                 struct table *elem;
4547
4548                 elem = TAILQ_FIRST(&p->tables);
4549                 if (!elem)
4550                         break;
4551
4552                 TAILQ_REMOVE(&p->tables, elem, node);
4553                 free(elem->fields);
4554                 free(elem->actions);
4555                 free(elem->default_action_data);
4556                 free(elem);
4557         }
4558
4559         /* Table types. */
4560         for ( ; ; ) {
4561                 struct table_type *elem;
4562
4563                 elem = TAILQ_FIRST(&p->table_types);
4564                 if (!elem)
4565                         break;
4566
4567                 TAILQ_REMOVE(&p->table_types, elem, node);
4568                 free(elem);
4569         }
4570 }
4571
4572 /*
4573  * Pipeline.
4574  */
4575 int
4576 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
4577 {
4578         struct rte_swx_pipeline *pipeline;
4579
4580         /* Check input parameters. */
4581         CHECK(p, EINVAL);
4582
4583         /* Memory allocation. */
4584         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
4585         CHECK(pipeline, ENOMEM);
4586
4587         /* Initialization. */
4588         TAILQ_INIT(&pipeline->struct_types);
4589         TAILQ_INIT(&pipeline->port_in_types);
4590         TAILQ_INIT(&pipeline->ports_in);
4591         TAILQ_INIT(&pipeline->port_out_types);
4592         TAILQ_INIT(&pipeline->ports_out);
4593         TAILQ_INIT(&pipeline->extern_types);
4594         TAILQ_INIT(&pipeline->extern_objs);
4595         TAILQ_INIT(&pipeline->extern_funcs);
4596         TAILQ_INIT(&pipeline->headers);
4597         TAILQ_INIT(&pipeline->actions);
4598         TAILQ_INIT(&pipeline->table_types);
4599         TAILQ_INIT(&pipeline->tables);
4600
4601         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
4602         pipeline->numa_node = numa_node;
4603
4604         *p = pipeline;
4605         return 0;
4606 }
4607
4608 void
4609 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
4610 {
4611         if (!p)
4612                 return;
4613
4614         free(p->instructions);
4615
4616         table_state_free(p);
4617         table_free(p);
4618         action_free(p);
4619         metadata_free(p);
4620         header_free(p);
4621         extern_func_free(p);
4622         extern_obj_free(p);
4623         port_out_free(p);
4624         port_in_free(p);
4625         struct_free(p);
4626
4627         free(p);
4628 }
4629
4630 int
4631 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
4632                                      const char **instructions,
4633                                      uint32_t n_instructions)
4634 {
4635         int err;
4636         uint32_t i;
4637
4638         err = instruction_config(p, NULL, instructions, n_instructions);
4639         if (err)
4640                 return err;
4641
4642         /* Thread instruction pointer reset. */
4643         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
4644                 struct thread *t = &p->threads[i];
4645
4646                 thread_ip_reset(p, t);
4647         }
4648
4649         return 0;
4650 }
4651
4652 int
4653 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
4654 {
4655         int status;
4656
4657         CHECK(p, EINVAL);
4658         CHECK(p->build_done == 0, EEXIST);
4659
4660         status = port_in_build(p);
4661         if (status)
4662                 goto error;
4663
4664         status = port_out_build(p);
4665         if (status)
4666                 goto error;
4667
4668         status = struct_build(p);
4669         if (status)
4670                 goto error;
4671
4672         status = extern_obj_build(p);
4673         if (status)
4674                 goto error;
4675
4676         status = extern_func_build(p);
4677         if (status)
4678                 goto error;
4679
4680         status = header_build(p);
4681         if (status)
4682                 goto error;
4683
4684         status = metadata_build(p);
4685         if (status)
4686                 goto error;
4687
4688         status = action_build(p);
4689         if (status)
4690                 goto error;
4691
4692         status = table_build(p);
4693         if (status)
4694                 goto error;
4695
4696         status = table_state_build(p);
4697         if (status)
4698                 goto error;
4699
4700         p->build_done = 1;
4701         return 0;
4702
4703 error:
4704         table_state_build_free(p);
4705         table_build_free(p);
4706         action_build_free(p);
4707         metadata_build_free(p);
4708         header_build_free(p);
4709         extern_func_build_free(p);
4710         extern_obj_build_free(p);
4711         port_out_build_free(p);
4712         port_in_build_free(p);
4713         struct_build_free(p);
4714
4715         return status;
4716 }
4717
4718 void
4719 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
4720 {
4721         uint32_t i;
4722
4723         for (i = 0; i < n_instructions; i++)
4724                 instr_exec(p);
4725 }
4726
4727 /*
4728  * Control.
4729  */
4730 int
4731 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
4732                                  struct rte_swx_table_state **table_state)
4733 {
4734         if (!p || !table_state || !p->build_done)
4735                 return -EINVAL;
4736
4737         *table_state = p->table_state;
4738         return 0;
4739 }
4740
4741 int
4742 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
4743                                  struct rte_swx_table_state *table_state)
4744 {
4745         if (!p || !table_state || !p->build_done)
4746                 return -EINVAL;
4747
4748         p->table_state = table_state;
4749         return 0;
4750 }