lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9 #include <arpa/inet.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_prefetch.h>
  13 #include <rte_byteorder.h>
  14
  15 #include "rte_swx_pipeline.h"
  16 #include "rte_swx_ctl.h"
  17
  18 #define CHECK(condition, err_code)                                             \
  19 do {                                                                           \
  20         if (!(condition))                                                      \
  21                 return -(err_code);                                            \
  22 } while (0)
  23
  24 #define CHECK_NAME(name, err_code)                                             \
  25         CHECK((name) && (name)[0], err_code)
  26
  27 #ifndef TRACE_LEVEL
  28 #define TRACE_LEVEL 0
  29 #endif
  30
  31 #if TRACE_LEVEL
  32 #define TRACE(...) printf(__VA_ARGS__)
  33 #else
  34 #define TRACE(...)
  35 #endif
  36
  37 #define ntoh64(x) rte_be_to_cpu_64(x)
  38 #define hton64(x) rte_cpu_to_be_64(x)
  39
  40 /*
  41  * Struct.
  42  */
  43 struct field {
  44         char name[RTE_SWX_NAME_SIZE];
  45         uint32_t n_bits;
  46         uint32_t offset;
  47 };
  48
  49 struct struct_type {
  50         TAILQ_ENTRY(struct_type) node;
  51         char name[RTE_SWX_NAME_SIZE];
  52         struct field *fields;
  53         uint32_t n_fields;
  54         uint32_t n_bits;
  55 };
  56
  57 TAILQ_HEAD(struct_type_tailq, struct_type);
  58
  59 /*
  60  * Input port.
  61  */
  62 struct port_in_type {
  63         TAILQ_ENTRY(port_in_type) node;
  64         char name[RTE_SWX_NAME_SIZE];
  65         struct rte_swx_port_in_ops ops;
  66 };
  67
  68 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  69
  70 struct port_in {
  71         TAILQ_ENTRY(port_in) node;
  72         struct port_in_type *type;
  73         void *obj;
  74         uint32_t id;
  75 };
  76
  77 TAILQ_HEAD(port_in_tailq, port_in);
  78
  79 struct port_in_runtime {
  80         rte_swx_port_in_pkt_rx_t pkt_rx;
  81         void *obj;
  82 };
  83
  84 /*
  85  * Output port.
  86  */
  87 struct port_out_type {
  88         TAILQ_ENTRY(port_out_type) node;
  89         char name[RTE_SWX_NAME_SIZE];
  90         struct rte_swx_port_out_ops ops;
  91 };
  92
  93 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  94
  95 struct port_out {
  96         TAILQ_ENTRY(port_out) node;
  97         struct port_out_type *type;
  98         void *obj;
  99         uint32_t id;
 100 };
 101
 102 TAILQ_HEAD(port_out_tailq, port_out);
 103
 104 struct port_out_runtime {
 105         rte_swx_port_out_pkt_tx_t pkt_tx;
 106         rte_swx_port_out_flush_t flush;
 107         void *obj;
 108 };
 109
 110 /*
 111  * Extern object.
 112  */
 113 struct extern_type_member_func {
 114         TAILQ_ENTRY(extern_type_member_func) node;
 115         char name[RTE_SWX_NAME_SIZE];
 116         rte_swx_extern_type_member_func_t func;
 117         uint32_t id;
 118 };
 119
 120 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 121
 122 struct extern_type {
 123         TAILQ_ENTRY(extern_type) node;
 124         char name[RTE_SWX_NAME_SIZE];
 125         struct struct_type *mailbox_struct_type;
 126         rte_swx_extern_type_constructor_t constructor;
 127         rte_swx_extern_type_destructor_t destructor;
 128         struct extern_type_member_func_tailq funcs;
 129         uint32_t n_funcs;
 130 };
 131
 132 TAILQ_HEAD(extern_type_tailq, extern_type);
 133
 134 struct extern_obj {
 135         TAILQ_ENTRY(extern_obj) node;
 136         char name[RTE_SWX_NAME_SIZE];
 137         struct extern_type *type;
 138         void *obj;
 139         uint32_t struct_id;
 140         uint32_t id;
 141 };
 142
 143 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 144
 145 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 146 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 147 #endif
 148
 149 struct extern_obj_runtime {
 150         void *obj;
 151         uint8_t *mailbox;
 152         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 153 };
 154
 155 /*
 156  * Extern function.
 157  */
 158 struct extern_func {
 159         TAILQ_ENTRY(extern_func) node;
 160         char name[RTE_SWX_NAME_SIZE];
 161         struct struct_type *mailbox_struct_type;
 162         rte_swx_extern_func_t func;
 163         uint32_t struct_id;
 164         uint32_t id;
 165 };
 166
 167 TAILQ_HEAD(extern_func_tailq, extern_func);
 168
 169 struct extern_func_runtime {
 170         uint8_t *mailbox;
 171         rte_swx_extern_func_t func;
 172 };
 173
 174 /*
 175  * Header.
 176  */
 177 struct header {
 178         TAILQ_ENTRY(header) node;
 179         char name[RTE_SWX_NAME_SIZE];
 180         struct struct_type *st;
 181         uint32_t struct_id;
 182         uint32_t id;
 183 };
 184
 185 TAILQ_HEAD(header_tailq, header);
 186
 187 struct header_runtime {
 188         uint8_t *ptr0;
 189 };
 190
 191 struct header_out_runtime {
 192         uint8_t *ptr0;
 193         uint8_t *ptr;
 194         uint32_t n_bytes;
 195 };
 196
 197 /*
 198  * Instruction.
 199  */
 200
 201 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 202  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 203  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 204  * when transferred to packet meta-data and in NBO when transferred to packet
 205  * headers.
 206  */
 207
 208 /* Notation conventions:
 209  *    -Header field: H = h.header.field (dst/src)
 210  *    -Meta-data field: M = m.field (dst/src)
 211  *    -Extern object mailbox field: E = e.field (dst/src)
 212  *    -Extern function mailbox field: F = f.field (dst/src)
 213  *    -Table action data field: T = t.field (src only)
 214  *    -Immediate value: I = 32-bit unsigned value (src only)
 215  */
 216
 217 enum instruction_type {
 218         /* rx m.port_in */
 219         INSTR_RX,
 220
 221         /* tx m.port_out */
 222         INSTR_TX,
 223
 224         /* extract h.header */
 225         INSTR_HDR_EXTRACT,
 226         INSTR_HDR_EXTRACT2,
 227         INSTR_HDR_EXTRACT3,
 228         INSTR_HDR_EXTRACT4,
 229         INSTR_HDR_EXTRACT5,
 230         INSTR_HDR_EXTRACT6,
 231         INSTR_HDR_EXTRACT7,
 232         INSTR_HDR_EXTRACT8,
 233
 234         /* emit h.header */
 235         INSTR_HDR_EMIT,
 236         INSTR_HDR_EMIT_TX,
 237         INSTR_HDR_EMIT2_TX,
 238         INSTR_HDR_EMIT3_TX,
 239         INSTR_HDR_EMIT4_TX,
 240         INSTR_HDR_EMIT5_TX,
 241         INSTR_HDR_EMIT6_TX,
 242         INSTR_HDR_EMIT7_TX,
 243         INSTR_HDR_EMIT8_TX,
 244
 245         /* validate h.header */
 246         INSTR_HDR_VALIDATE,
 247
 248         /* invalidate h.header */
 249         INSTR_HDR_INVALIDATE,
 250
 251         /* mov dst src
 252          * dst = src
 253          * dst = HMEF, src = HMEFTI
 254          */
 255         INSTR_MOV,   /* dst = MEF, src = MEFT */
 256         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 257         INSTR_MOV_I, /* dst = HMEF, src = I */
 258
 259         /* dma h.header t.field
 260          * memcpy(h.header, t.field, sizeof(h.header))
 261          */
 262         INSTR_DMA_HT,
 263         INSTR_DMA_HT2,
 264         INSTR_DMA_HT3,
 265         INSTR_DMA_HT4,
 266         INSTR_DMA_HT5,
 267         INSTR_DMA_HT6,
 268         INSTR_DMA_HT7,
 269         INSTR_DMA_HT8,
 270
 271         /* add dst src
 272          * dst += src
 273          * dst = HMEF, src = HMEFTI
 274          */
 275         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 276         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 277         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 278         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 279         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 280         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 281
 282         /* sub dst src
 283          * dst -= src
 284          * dst = HMEF, src = HMEFTI
 285          */
 286         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 287         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 288         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 289         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 290         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 291         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 292
 293         /* ckadd dst src
 294          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 295          * dst = H, src = {H, h.header}
 296          */
 297         INSTR_ALU_CKADD_FIELD,    /* src = H */
 298         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 299         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 300
 301         /* cksub dst src
 302          * dst = dst '- src
 303          * dst = H, src = H
 304          */
 305         INSTR_ALU_CKSUB_FIELD,
 306
 307         /* and dst src
 308          * dst &= src
 309          * dst = HMEF, src = HMEFTI
 310          */
 311         INSTR_ALU_AND,   /* dst = MEF, src = MEFT */
 312         INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 313         INSTR_ALU_AND_I, /* dst = HMEF, src = I */
 314
 315         /* or dst src
 316          * dst |= src
 317          * dst = HMEF, src = HMEFTI
 318          */
 319         INSTR_ALU_OR,   /* dst = MEF, src = MEFT */
 320         INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 321         INSTR_ALU_OR_I, /* dst = HMEF, src = I */
 322
 323         /* xor dst src
 324          * dst ^= src
 325          * dst = HMEF, src = HMEFTI
 326          */
 327         INSTR_ALU_XOR,   /* dst = MEF, src = MEFT */
 328         INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 329         INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
 330
 331         /* shl dst src
 332          * dst <<= src
 333          * dst = HMEF, src = HMEFTI
 334          */
 335         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 336         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 337         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 338         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 339         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 340         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 341
 342         /* shr dst src
 343          * dst >>= src
 344          * dst = HMEF, src = HMEFTI
 345          */
 346         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 347         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 348         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 349         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 350         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 351         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 352
 353         /* table TABLE */
 354         INSTR_TABLE,
 355 };
 356
 357 struct instr_operand {
 358         uint8_t struct_id;
 359         uint8_t n_bits;
 360         uint8_t offset;
 361         uint8_t pad;
 362 };
 363
 364 struct instr_io {
 365         struct {
 366                 uint8_t offset;
 367                 uint8_t n_bits;
 368                 uint8_t pad[2];
 369         } io;
 370
 371         struct {
 372                 uint8_t header_id[8];
 373                 uint8_t struct_id[8];
 374                 uint8_t n_bytes[8];
 375         } hdr;
 376 };
 377
 378 struct instr_hdr_validity {
 379         uint8_t header_id;
 380 };
 381
 382 struct instr_table {
 383         uint8_t table_id;
 384 };
 385
 386 struct instr_dst_src {
 387         struct instr_operand dst;
 388         union {
 389                 struct instr_operand src;
 390                 uint32_t src_val;
 391         };
 392 };
 393
 394 struct instr_dma {
 395         struct {
 396                 uint8_t header_id[8];
 397                 uint8_t struct_id[8];
 398         } dst;
 399
 400         struct {
 401                 uint8_t offset[8];
 402         } src;
 403
 404         uint16_t n_bytes[8];
 405 };
 406
 407 struct instruction {
 408         enum instruction_type type;
 409         union {
 410                 struct instr_io io;
 411                 struct instr_hdr_validity valid;
 412                 struct instr_dst_src mov;
 413                 struct instr_dma dma;
 414                 struct instr_dst_src alu;
 415                 struct instr_table table;
 416         };
 417 };
 418
 419 struct instruction_data {
 420         char label[RTE_SWX_NAME_SIZE];
 421         char jmp_label[RTE_SWX_NAME_SIZE];
 422         uint32_t n_users; /* user = jmp instruction to this instruction. */
 423         int invalid;
 424 };
 425
 426 /*
 427  * Action.
 428  */
 429 struct action {
 430         TAILQ_ENTRY(action) node;
 431         char name[RTE_SWX_NAME_SIZE];
 432         struct struct_type *st;
 433         struct instruction *instructions;
 434         uint32_t n_instructions;
 435         uint32_t id;
 436 };
 437
 438 TAILQ_HEAD(action_tailq, action);
 439
 440 /*
 441  * Table.
 442  */
 443 struct table_type {
 444         TAILQ_ENTRY(table_type) node;
 445         char name[RTE_SWX_NAME_SIZE];
 446         enum rte_swx_table_match_type match_type;
 447         struct rte_swx_table_ops ops;
 448 };
 449
 450 TAILQ_HEAD(table_type_tailq, table_type);
 451
 452 struct match_field {
 453         enum rte_swx_table_match_type match_type;
 454         struct field *field;
 455 };
 456
 457 struct table {
 458         TAILQ_ENTRY(table) node;
 459         char name[RTE_SWX_NAME_SIZE];
 460         char args[RTE_SWX_NAME_SIZE];
 461         struct table_type *type; /* NULL when n_fields == 0. */
 462
 463         /* Match. */
 464         struct match_field *fields;
 465         uint32_t n_fields;
 466         int is_header; /* Only valid when n_fields > 0. */
 467         struct header *header; /* Only valid when n_fields > 0. */
 468
 469         /* Action. */
 470         struct action **actions;
 471         struct action *default_action;
 472         uint8_t *default_action_data;
 473         uint32_t n_actions;
 474         int default_action_is_const;
 475         uint32_t action_data_size_max;
 476
 477         uint32_t size;
 478         uint32_t id;
 479 };
 480
 481 TAILQ_HEAD(table_tailq, table);
 482
 483 struct table_runtime {
 484         rte_swx_table_lookup_t func;
 485         void *mailbox;
 486         uint8_t **key;
 487 };
 488
 489 /*
 490  * Pipeline.
 491  */
 492 struct thread {
 493         /* Packet. */
 494         struct rte_swx_pkt pkt;
 495         uint8_t *ptr;
 496
 497         /* Structures. */
 498         uint8_t **structs;
 499
 500         /* Packet headers. */
 501         struct header_runtime *headers; /* Extracted or generated headers. */
 502         struct header_out_runtime *headers_out; /* Emitted headers. */
 503         uint8_t *header_storage;
 504         uint8_t *header_out_storage;
 505         uint64_t valid_headers;
 506         uint32_t n_headers_out;
 507
 508         /* Packet meta-data. */
 509         uint8_t *metadata;
 510
 511         /* Tables. */
 512         struct table_runtime *tables;
 513         struct rte_swx_table_state *table_state;
 514         uint64_t action_id;
 515         int hit; /* 0 = Miss, 1 = Hit. */
 516
 517         /* Extern objects and functions. */
 518         struct extern_obj_runtime *extern_objs;
 519         struct extern_func_runtime *extern_funcs;
 520
 521         /* Instructions. */
 522         struct instruction *ip;
 523         struct instruction *ret;
 524 };
 525
 526 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 527 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 528 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 529
 530 #define ALU(thread, ip, operator)  \
 531 {                                                                              \
 532         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 533         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 534         uint64_t dst64 = *dst64_ptr;                                           \
 535         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 536         uint64_t dst = dst64 & dst64_mask;                                     \
 537                                                                                \
 538         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 539         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 540         uint64_t src64 = *src64_ptr;                                           \
 541         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 542         uint64_t src = src64 & src64_mask;                                     \
 543                                                                                \
 544         uint64_t result = dst operator src;                                    \
 545                                                                                \
 546         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 547 }
 548
 549 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 550
 551 #define ALU_S(thread, ip, operator)  \
 552 {                                                                              \
 553         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 554         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 555         uint64_t dst64 = *dst64_ptr;                                           \
 556         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 557         uint64_t dst = dst64 & dst64_mask;                                     \
 558                                                                                \
 559         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 560         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 561         uint64_t src64 = *src64_ptr;                                           \
 562         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 563                                                                                \
 564         uint64_t result = dst operator src;                                    \
 565                                                                                \
 566         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 567 }
 568
 569 #define ALU_MH ALU_S
 570
 571 #define ALU_HM(thread, ip, operator)  \
 572 {                                                                              \
 573         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 574         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 575         uint64_t dst64 = *dst64_ptr;                                           \
 576         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 577         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 578                                                                                \
 579         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 580         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 581         uint64_t src64 = *src64_ptr;                                           \
 582         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 583         uint64_t src = src64 & src64_mask;                                     \
 584                                                                                \
 585         uint64_t result = dst operator src;                                    \
 586         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 587                                                                                \
 588         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 589 }
 590
 591 #define ALU_HH(thread, ip, operator)  \
 592 {                                                                              \
 593         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 594         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 595         uint64_t dst64 = *dst64_ptr;                                           \
 596         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 597         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 598                                                                                \
 599         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 600         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 601         uint64_t src64 = *src64_ptr;                                           \
 602         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 603                                                                                \
 604         uint64_t result = dst operator src;                                    \
 605         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 606                                                                                \
 607         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 608 }
 609
 610 #else
 611
 612 #define ALU_S ALU
 613 #define ALU_MH ALU
 614 #define ALU_HM ALU
 615 #define ALU_HH ALU
 616
 617 #endif
 618
 619 #define ALU_I(thread, ip, operator)  \
 620 {                                                                              \
 621         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 622         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 623         uint64_t dst64 = *dst64_ptr;                                           \
 624         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 625         uint64_t dst = dst64 & dst64_mask;                                     \
 626                                                                                \
 627         uint64_t src = (ip)->alu.src_val;                                      \
 628                                                                                \
 629         uint64_t result = dst operator src;                                    \
 630                                                                                \
 631         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 632 }
 633
 634 #define ALU_MI ALU_I
 635
 636 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 637
 638 #define ALU_HI(thread, ip, operator)  \
 639 {                                                                              \
 640         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 641         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 642         uint64_t dst64 = *dst64_ptr;                                           \
 643         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 644         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 645                                                                                \
 646         uint64_t src = (ip)->alu.src_val;                                      \
 647                                                                                \
 648         uint64_t result = dst operator src;                                    \
 649         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 650                                                                                \
 651         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 652 }
 653
 654 #else
 655
 656 #define ALU_HI ALU_I
 657
 658 #endif
 659
 660 #define MOV(thread, ip)  \
 661 {                                                                              \
 662         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 663         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 664         uint64_t dst64 = *dst64_ptr;                                           \
 665         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 666                                                                                \
 667         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 668         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 669         uint64_t src64 = *src64_ptr;                                           \
 670         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
 671         uint64_t src = src64 & src64_mask;                                     \
 672                                                                                \
 673         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 674 }
 675
 676 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 677
 678 #define MOV_S(thread, ip)  \
 679 {                                                                              \
 680         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 681         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 682         uint64_t dst64 = *dst64_ptr;                                           \
 683         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 684                                                                                \
 685         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 686         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 687         uint64_t src64 = *src64_ptr;                                           \
 688         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
 689                                                                                \
 690         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 691 }
 692
 693 #else
 694
 695 #define MOV_S MOV
 696
 697 #endif
 698
 699 #define MOV_I(thread, ip)  \
 700 {                                                                              \
 701         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 702         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 703         uint64_t dst64 = *dst64_ptr;                                           \
 704         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 705                                                                                \
 706         uint64_t src = (ip)->mov.src_val;                                      \
 707                                                                                \
 708         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 709 }
 710
 711 #define METADATA_READ(thread, offset, n_bits)                                  \
 712 ({                                                                             \
 713         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 714         uint64_t m64 = *m64_ptr;                                               \
 715         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 716         (m64 & m64_mask);                                                      \
 717 })
 718
 719 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
 720 {                                                                              \
 721         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 722         uint64_t m64 = *m64_ptr;                                               \
 723         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 724                                                                                \
 725         uint64_t m_new = value;                                                \
 726                                                                                \
 727         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
 728 }
 729
 730 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 731 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 732 #endif
 733
 734 struct rte_swx_pipeline {
 735         struct struct_type_tailq struct_types;
 736         struct port_in_type_tailq port_in_types;
 737         struct port_in_tailq ports_in;
 738         struct port_out_type_tailq port_out_types;
 739         struct port_out_tailq ports_out;
 740         struct extern_type_tailq extern_types;
 741         struct extern_obj_tailq extern_objs;
 742         struct extern_func_tailq extern_funcs;
 743         struct header_tailq headers;
 744         struct struct_type *metadata_st;
 745         uint32_t metadata_struct_id;
 746         struct action_tailq actions;
 747         struct table_type_tailq table_types;
 748         struct table_tailq tables;
 749
 750         struct port_in_runtime *in;
 751         struct port_out_runtime *out;
 752         struct instruction **action_instructions;
 753         struct rte_swx_table_state *table_state;
 754         struct instruction *instructions;
 755         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 756
 757         uint32_t n_structs;
 758         uint32_t n_ports_in;
 759         uint32_t n_ports_out;
 760         uint32_t n_extern_objs;
 761         uint32_t n_extern_funcs;
 762         uint32_t n_actions;
 763         uint32_t n_tables;
 764         uint32_t n_headers;
 765         uint32_t thread_id;
 766         uint32_t port_id;
 767         uint32_t n_instructions;
 768         int build_done;
 769         int numa_node;
 770 };
 771
 772 /*
 773  * Struct.
 774  */
 775 static struct struct_type *
 776 struct_type_find(struct rte_swx_pipeline *p, const char *name)
 777 {
 778         struct struct_type *elem;
 779
 780         TAILQ_FOREACH(elem, &p->struct_types, node)
 781                 if (strcmp(elem->name, name) == 0)
 782                         return elem;
 783
 784         return NULL;
 785 }
 786
 787 static struct field *
 788 struct_type_field_find(struct struct_type *st, const char *name)
 789 {
 790         uint32_t i;
 791
 792         for (i = 0; i < st->n_fields; i++) {
 793                 struct field *f = &st->fields[i];
 794
 795                 if (strcmp(f->name, name) == 0)
 796                         return f;
 797         }
 798
 799         return NULL;
 800 }
 801
 802 int
 803 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 804                                       const char *name,
 805                                       struct rte_swx_field_params *fields,
 806                                       uint32_t n_fields)
 807 {
 808         struct struct_type *st;
 809         uint32_t i;
 810
 811         CHECK(p, EINVAL);
 812         CHECK_NAME(name, EINVAL);
 813         CHECK(fields, EINVAL);
 814         CHECK(n_fields, EINVAL);
 815
 816         for (i = 0; i < n_fields; i++) {
 817                 struct rte_swx_field_params *f = &fields[i];
 818                 uint32_t j;
 819
 820                 CHECK_NAME(f->name, EINVAL);
 821                 CHECK(f->n_bits, EINVAL);
 822                 CHECK(f->n_bits <= 64, EINVAL);
 823                 CHECK((f->n_bits & 7) == 0, EINVAL);
 824
 825                 for (j = 0; j < i; j++) {
 826                         struct rte_swx_field_params *f_prev = &fields[j];
 827
 828                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 829                 }
 830         }
 831
 832         CHECK(!struct_type_find(p, name), EEXIST);
 833
 834         /* Node allocation. */
 835         st = calloc(1, sizeof(struct struct_type));
 836         CHECK(st, ENOMEM);
 837
 838         st->fields = calloc(n_fields, sizeof(struct field));
 839         if (!st->fields) {
 840                 free(st);
 841                 CHECK(0, ENOMEM);
 842         }
 843
 844         /* Node initialization. */
 845         strcpy(st->name, name);
 846         for (i = 0; i < n_fields; i++) {
 847                 struct field *dst = &st->fields[i];
 848                 struct rte_swx_field_params *src = &fields[i];
 849
 850                 strcpy(dst->name, src->name);
 851                 dst->n_bits = src->n_bits;
 852                 dst->offset = st->n_bits;
 853
 854                 st->n_bits += src->n_bits;
 855         }
 856         st->n_fields = n_fields;
 857
 858         /* Node add to tailq. */
 859         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 860
 861         return 0;
 862 }
 863
 864 static int
 865 struct_build(struct rte_swx_pipeline *p)
 866 {
 867         uint32_t i;
 868
 869         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 870                 struct thread *t = &p->threads[i];
 871
 872                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 873                 CHECK(t->structs, ENOMEM);
 874         }
 875
 876         return 0;
 877 }
 878
 879 static void
 880 struct_build_free(struct rte_swx_pipeline *p)
 881 {
 882         uint32_t i;
 883
 884         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 885                 struct thread *t = &p->threads[i];
 886
 887                 free(t->structs);
 888                 t->structs = NULL;
 889         }
 890 }
 891
 892 static void
 893 struct_free(struct rte_swx_pipeline *p)
 894 {
 895         struct_build_free(p);
 896
 897         /* Struct types. */
 898         for ( ; ; ) {
 899                 struct struct_type *elem;
 900
 901                 elem = TAILQ_FIRST(&p->struct_types);
 902                 if (!elem)
 903                         break;
 904
 905                 TAILQ_REMOVE(&p->struct_types, elem, node);
 906                 free(elem->fields);
 907                 free(elem);
 908         }
 909 }
 910
 911 /*
 912  * Input port.
 913  */
 914 static struct port_in_type *
 915 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 916 {
 917         struct port_in_type *elem;
 918
 919         if (!name)
 920                 return NULL;
 921
 922         TAILQ_FOREACH(elem, &p->port_in_types, node)
 923                 if (strcmp(elem->name, name) == 0)
 924                         return elem;
 925
 926         return NULL;
 927 }
 928
 929 int
 930 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 931                                        const char *name,
 932                                        struct rte_swx_port_in_ops *ops)
 933 {
 934         struct port_in_type *elem;
 935
 936         CHECK(p, EINVAL);
 937         CHECK_NAME(name, EINVAL);
 938         CHECK(ops, EINVAL);
 939         CHECK(ops->create, EINVAL);
 940         CHECK(ops->free, EINVAL);
 941         CHECK(ops->pkt_rx, EINVAL);
 942         CHECK(ops->stats_read, EINVAL);
 943
 944         CHECK(!port_in_type_find(p, name), EEXIST);
 945
 946         /* Node allocation. */
 947         elem = calloc(1, sizeof(struct port_in_type));
 948         CHECK(elem, ENOMEM);
 949
 950         /* Node initialization. */
 951         strcpy(elem->name, name);
 952         memcpy(&elem->ops, ops, sizeof(*ops));
 953
 954         /* Node add to tailq. */
 955         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 956
 957         return 0;
 958 }
 959
 960 static struct port_in *
 961 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 962 {
 963         struct port_in *port;
 964
 965         TAILQ_FOREACH(port, &p->ports_in, node)
 966                 if (port->id == port_id)
 967                         return port;
 968
 969         return NULL;
 970 }
 971
 972 int
 973 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 974                                 uint32_t port_id,
 975                                 const char *port_type_name,
 976                                 void *args)
 977 {
 978         struct port_in_type *type = NULL;
 979         struct port_in *port = NULL;
 980         void *obj = NULL;
 981
 982         CHECK(p, EINVAL);
 983
 984         CHECK(!port_in_find(p, port_id), EINVAL);
 985
 986         CHECK_NAME(port_type_name, EINVAL);
 987         type = port_in_type_find(p, port_type_name);
 988         CHECK(type, EINVAL);
 989
 990         obj = type->ops.create(args);
 991         CHECK(obj, ENODEV);
 992
 993         /* Node allocation. */
 994         port = calloc(1, sizeof(struct port_in));
 995         CHECK(port, ENOMEM);
 996
 997         /* Node initialization. */
 998         port->type = type;
 999         port->obj = obj;
1000         port->id = port_id;
1001
1002         /* Node add to tailq. */
1003         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1004         if (p->n_ports_in < port_id + 1)
1005                 p->n_ports_in = port_id + 1;
1006
1007         return 0;
1008 }
1009
1010 static int
1011 port_in_build(struct rte_swx_pipeline *p)
1012 {
1013         struct port_in *port;
1014         uint32_t i;
1015
1016         CHECK(p->n_ports_in, EINVAL);
1017         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1018
1019         for (i = 0; i < p->n_ports_in; i++)
1020                 CHECK(port_in_find(p, i), EINVAL);
1021
1022         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1023         CHECK(p->in, ENOMEM);
1024
1025         TAILQ_FOREACH(port, &p->ports_in, node) {
1026                 struct port_in_runtime *in = &p->in[port->id];
1027
1028                 in->pkt_rx = port->type->ops.pkt_rx;
1029                 in->obj = port->obj;
1030         }
1031
1032         return 0;
1033 }
1034
1035 static void
1036 port_in_build_free(struct rte_swx_pipeline *p)
1037 {
1038         free(p->in);
1039         p->in = NULL;
1040 }
1041
1042 static void
1043 port_in_free(struct rte_swx_pipeline *p)
1044 {
1045         port_in_build_free(p);
1046
1047         /* Input ports. */
1048         for ( ; ; ) {
1049                 struct port_in *port;
1050
1051                 port = TAILQ_FIRST(&p->ports_in);
1052                 if (!port)
1053                         break;
1054
1055                 TAILQ_REMOVE(&p->ports_in, port, node);
1056                 port->type->ops.free(port->obj);
1057                 free(port);
1058         }
1059
1060         /* Input port types. */
1061         for ( ; ; ) {
1062                 struct port_in_type *elem;
1063
1064                 elem = TAILQ_FIRST(&p->port_in_types);
1065                 if (!elem)
1066                         break;
1067
1068                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1069                 free(elem);
1070         }
1071 }
1072
1073 /*
1074  * Output port.
1075  */
1076 static struct port_out_type *
1077 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1078 {
1079         struct port_out_type *elem;
1080
1081         if (!name)
1082                 return NULL;
1083
1084         TAILQ_FOREACH(elem, &p->port_out_types, node)
1085                 if (!strcmp(elem->name, name))
1086                         return elem;
1087
1088         return NULL;
1089 }
1090
1091 int
1092 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1093                                         const char *name,
1094                                         struct rte_swx_port_out_ops *ops)
1095 {
1096         struct port_out_type *elem;
1097
1098         CHECK(p, EINVAL);
1099         CHECK_NAME(name, EINVAL);
1100         CHECK(ops, EINVAL);
1101         CHECK(ops->create, EINVAL);
1102         CHECK(ops->free, EINVAL);
1103         CHECK(ops->pkt_tx, EINVAL);
1104         CHECK(ops->stats_read, EINVAL);
1105
1106         CHECK(!port_out_type_find(p, name), EEXIST);
1107
1108         /* Node allocation. */
1109         elem = calloc(1, sizeof(struct port_out_type));
1110         CHECK(elem, ENOMEM);
1111
1112         /* Node initialization. */
1113         strcpy(elem->name, name);
1114         memcpy(&elem->ops, ops, sizeof(*ops));
1115
1116         /* Node add to tailq. */
1117         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1118
1119         return 0;
1120 }
1121
1122 static struct port_out *
1123 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1124 {
1125         struct port_out *port;
1126
1127         TAILQ_FOREACH(port, &p->ports_out, node)
1128                 if (port->id == port_id)
1129                         return port;
1130
1131         return NULL;
1132 }
1133
1134 int
1135 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1136                                  uint32_t port_id,
1137                                  const char *port_type_name,
1138                                  void *args)
1139 {
1140         struct port_out_type *type = NULL;
1141         struct port_out *port = NULL;
1142         void *obj = NULL;
1143
1144         CHECK(p, EINVAL);
1145
1146         CHECK(!port_out_find(p, port_id), EINVAL);
1147
1148         CHECK_NAME(port_type_name, EINVAL);
1149         type = port_out_type_find(p, port_type_name);
1150         CHECK(type, EINVAL);
1151
1152         obj = type->ops.create(args);
1153         CHECK(obj, ENODEV);
1154
1155         /* Node allocation. */
1156         port = calloc(1, sizeof(struct port_out));
1157         CHECK(port, ENOMEM);
1158
1159         /* Node initialization. */
1160         port->type = type;
1161         port->obj = obj;
1162         port->id = port_id;
1163
1164         /* Node add to tailq. */
1165         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1166         if (p->n_ports_out < port_id + 1)
1167                 p->n_ports_out = port_id + 1;
1168
1169         return 0;
1170 }
1171
1172 static int
1173 port_out_build(struct rte_swx_pipeline *p)
1174 {
1175         struct port_out *port;
1176         uint32_t i;
1177
1178         CHECK(p->n_ports_out, EINVAL);
1179
1180         for (i = 0; i < p->n_ports_out; i++)
1181                 CHECK(port_out_find(p, i), EINVAL);
1182
1183         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1184         CHECK(p->out, ENOMEM);
1185
1186         TAILQ_FOREACH(port, &p->ports_out, node) {
1187                 struct port_out_runtime *out = &p->out[port->id];
1188
1189                 out->pkt_tx = port->type->ops.pkt_tx;
1190                 out->flush = port->type->ops.flush;
1191                 out->obj = port->obj;
1192         }
1193
1194         return 0;
1195 }
1196
1197 static void
1198 port_out_build_free(struct rte_swx_pipeline *p)
1199 {
1200         free(p->out);
1201         p->out = NULL;
1202 }
1203
1204 static void
1205 port_out_free(struct rte_swx_pipeline *p)
1206 {
1207         port_out_build_free(p);
1208
1209         /* Output ports. */
1210         for ( ; ; ) {
1211                 struct port_out *port;
1212
1213                 port = TAILQ_FIRST(&p->ports_out);
1214                 if (!port)
1215                         break;
1216
1217                 TAILQ_REMOVE(&p->ports_out, port, node);
1218                 port->type->ops.free(port->obj);
1219                 free(port);
1220         }
1221
1222         /* Output port types. */
1223         for ( ; ; ) {
1224                 struct port_out_type *elem;
1225
1226                 elem = TAILQ_FIRST(&p->port_out_types);
1227                 if (!elem)
1228                         break;
1229
1230                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1231                 free(elem);
1232         }
1233 }
1234
1235 /*
1236  * Extern object.
1237  */
1238 static struct extern_type *
1239 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1240 {
1241         struct extern_type *elem;
1242
1243         TAILQ_FOREACH(elem, &p->extern_types, node)
1244                 if (strcmp(elem->name, name) == 0)
1245                         return elem;
1246
1247         return NULL;
1248 }
1249
1250 static struct extern_type_member_func *
1251 extern_type_member_func_find(struct extern_type *type, const char *name)
1252 {
1253         struct extern_type_member_func *elem;
1254
1255         TAILQ_FOREACH(elem, &type->funcs, node)
1256                 if (strcmp(elem->name, name) == 0)
1257                         return elem;
1258
1259         return NULL;
1260 }
1261
1262 static struct extern_obj *
1263 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1264 {
1265         struct extern_obj *elem;
1266
1267         TAILQ_FOREACH(elem, &p->extern_objs, node)
1268                 if (strcmp(elem->name, name) == 0)
1269                         return elem;
1270
1271         return NULL;
1272 }
1273
1274 static struct field *
1275 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1276                                const char *name,
1277                                struct extern_obj **object)
1278 {
1279         struct extern_obj *obj;
1280         struct field *f;
1281         char *obj_name, *field_name;
1282
1283         if ((name[0] != 'e') || (name[1] != '.'))
1284                 return NULL;
1285
1286         obj_name = strdup(&name[2]);
1287         if (!obj_name)
1288                 return NULL;
1289
1290         field_name = strchr(obj_name, '.');
1291         if (!field_name) {
1292                 free(obj_name);
1293                 return NULL;
1294         }
1295
1296         *field_name = 0;
1297         field_name++;
1298
1299         obj = extern_obj_find(p, obj_name);
1300         if (!obj) {
1301                 free(obj_name);
1302                 return NULL;
1303         }
1304
1305         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1306         if (!f) {
1307                 free(obj_name);
1308                 return NULL;
1309         }
1310
1311         if (object)
1312                 *object = obj;
1313
1314         free(obj_name);
1315         return f;
1316 }
1317
1318 int
1319 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1320         const char *name,
1321         const char *mailbox_struct_type_name,
1322         rte_swx_extern_type_constructor_t constructor,
1323         rte_swx_extern_type_destructor_t destructor)
1324 {
1325         struct extern_type *elem;
1326         struct struct_type *mailbox_struct_type;
1327
1328         CHECK(p, EINVAL);
1329
1330         CHECK_NAME(name, EINVAL);
1331         CHECK(!extern_type_find(p, name), EEXIST);
1332
1333         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1334         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1335         CHECK(mailbox_struct_type, EINVAL);
1336
1337         CHECK(constructor, EINVAL);
1338         CHECK(destructor, EINVAL);
1339
1340         /* Node allocation. */
1341         elem = calloc(1, sizeof(struct extern_type));
1342         CHECK(elem, ENOMEM);
1343
1344         /* Node initialization. */
1345         strcpy(elem->name, name);
1346         elem->mailbox_struct_type = mailbox_struct_type;
1347         elem->constructor = constructor;
1348         elem->destructor = destructor;
1349         TAILQ_INIT(&elem->funcs);
1350
1351         /* Node add to tailq. */
1352         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1353
1354         return 0;
1355 }
1356
1357 int
1358 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1359         const char *extern_type_name,
1360         const char *name,
1361         rte_swx_extern_type_member_func_t member_func)
1362 {
1363         struct extern_type *type;
1364         struct extern_type_member_func *type_member;
1365
1366         CHECK(p, EINVAL);
1367
1368         CHECK(extern_type_name, EINVAL);
1369         type = extern_type_find(p, extern_type_name);
1370         CHECK(type, EINVAL);
1371         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1372
1373         CHECK(name, EINVAL);
1374         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1375
1376         CHECK(member_func, EINVAL);
1377
1378         /* Node allocation. */
1379         type_member = calloc(1, sizeof(struct extern_type_member_func));
1380         CHECK(type_member, ENOMEM);
1381
1382         /* Node initialization. */
1383         strcpy(type_member->name, name);
1384         type_member->func = member_func;
1385         type_member->id = type->n_funcs;
1386
1387         /* Node add to tailq. */
1388         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1389         type->n_funcs++;
1390
1391         return 0;
1392 }
1393
1394 int
1395 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1396                                       const char *extern_type_name,
1397                                       const char *name,
1398                                       const char *args)
1399 {
1400         struct extern_type *type;
1401         struct extern_obj *obj;
1402         void *obj_handle;
1403
1404         CHECK(p, EINVAL);
1405
1406         CHECK_NAME(extern_type_name, EINVAL);
1407         type = extern_type_find(p, extern_type_name);
1408         CHECK(type, EINVAL);
1409
1410         CHECK_NAME(name, EINVAL);
1411         CHECK(!extern_obj_find(p, name), EEXIST);
1412
1413         /* Node allocation. */
1414         obj = calloc(1, sizeof(struct extern_obj));
1415         CHECK(obj, ENOMEM);
1416
1417         /* Object construction. */
1418         obj_handle = type->constructor(args);
1419         if (!obj_handle) {
1420                 free(obj);
1421                 CHECK(0, ENODEV);
1422         }
1423
1424         /* Node initialization. */
1425         strcpy(obj->name, name);
1426         obj->type = type;
1427         obj->obj = obj_handle;
1428         obj->struct_id = p->n_structs;
1429         obj->id = p->n_extern_objs;
1430
1431         /* Node add to tailq. */
1432         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1433         p->n_extern_objs++;
1434         p->n_structs++;
1435
1436         return 0;
1437 }
1438
1439 static int
1440 extern_obj_build(struct rte_swx_pipeline *p)
1441 {
1442         uint32_t i;
1443
1444         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1445                 struct thread *t = &p->threads[i];
1446                 struct extern_obj *obj;
1447
1448                 t->extern_objs = calloc(p->n_extern_objs,
1449                                         sizeof(struct extern_obj_runtime));
1450                 CHECK(t->extern_objs, ENOMEM);
1451
1452                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1453                         struct extern_obj_runtime *r =
1454                                 &t->extern_objs[obj->id];
1455                         struct extern_type_member_func *func;
1456                         uint32_t mailbox_size =
1457                                 obj->type->mailbox_struct_type->n_bits / 8;
1458
1459                         r->obj = obj->obj;
1460
1461                         r->mailbox = calloc(1, mailbox_size);
1462                         CHECK(r->mailbox, ENOMEM);
1463
1464                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1465                                 r->funcs[func->id] = func->func;
1466
1467                         t->structs[obj->struct_id] = r->mailbox;
1468                 }
1469         }
1470
1471         return 0;
1472 }
1473
1474 static void
1475 extern_obj_build_free(struct rte_swx_pipeline *p)
1476 {
1477         uint32_t i;
1478
1479         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1480                 struct thread *t = &p->threads[i];
1481                 uint32_t j;
1482
1483                 if (!t->extern_objs)
1484                         continue;
1485
1486                 for (j = 0; j < p->n_extern_objs; j++) {
1487                         struct extern_obj_runtime *r = &t->extern_objs[j];
1488
1489                         free(r->mailbox);
1490                 }
1491
1492                 free(t->extern_objs);
1493                 t->extern_objs = NULL;
1494         }
1495 }
1496
1497 static void
1498 extern_obj_free(struct rte_swx_pipeline *p)
1499 {
1500         extern_obj_build_free(p);
1501
1502         /* Extern objects. */
1503         for ( ; ; ) {
1504                 struct extern_obj *elem;
1505
1506                 elem = TAILQ_FIRST(&p->extern_objs);
1507                 if (!elem)
1508                         break;
1509
1510                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1511                 if (elem->obj)
1512                         elem->type->destructor(elem->obj);
1513                 free(elem);
1514         }
1515
1516         /* Extern types. */
1517         for ( ; ; ) {
1518                 struct extern_type *elem;
1519
1520                 elem = TAILQ_FIRST(&p->extern_types);
1521                 if (!elem)
1522                         break;
1523
1524                 TAILQ_REMOVE(&p->extern_types, elem, node);
1525
1526                 for ( ; ; ) {
1527                         struct extern_type_member_func *func;
1528
1529                         func = TAILQ_FIRST(&elem->funcs);
1530                         if (!func)
1531                                 break;
1532
1533                         TAILQ_REMOVE(&elem->funcs, func, node);
1534                         free(func);
1535                 }
1536
1537                 free(elem);
1538         }
1539 }
1540
1541 /*
1542  * Extern function.
1543  */
1544 static struct extern_func *
1545 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1546 {
1547         struct extern_func *elem;
1548
1549         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1550                 if (strcmp(elem->name, name) == 0)
1551                         return elem;
1552
1553         return NULL;
1554 }
1555
1556 static struct field *
1557 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1558                                 const char *name,
1559                                 struct extern_func **function)
1560 {
1561         struct extern_func *func;
1562         struct field *f;
1563         char *func_name, *field_name;
1564
1565         if ((name[0] != 'f') || (name[1] != '.'))
1566                 return NULL;
1567
1568         func_name = strdup(&name[2]);
1569         if (!func_name)
1570                 return NULL;
1571
1572         field_name = strchr(func_name, '.');
1573         if (!field_name) {
1574                 free(func_name);
1575                 return NULL;
1576         }
1577
1578         *field_name = 0;
1579         field_name++;
1580
1581         func = extern_func_find(p, func_name);
1582         if (!func) {
1583                 free(func_name);
1584                 return NULL;
1585         }
1586
1587         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1588         if (!f) {
1589                 free(func_name);
1590                 return NULL;
1591         }
1592
1593         if (function)
1594                 *function = func;
1595
1596         free(func_name);
1597         return f;
1598 }
1599
1600 int
1601 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1602                                       const char *name,
1603                                       const char *mailbox_struct_type_name,
1604                                       rte_swx_extern_func_t func)
1605 {
1606         struct extern_func *f;
1607         struct struct_type *mailbox_struct_type;
1608
1609         CHECK(p, EINVAL);
1610
1611         CHECK_NAME(name, EINVAL);
1612         CHECK(!extern_func_find(p, name), EEXIST);
1613
1614         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1615         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1616         CHECK(mailbox_struct_type, EINVAL);
1617
1618         CHECK(func, EINVAL);
1619
1620         /* Node allocation. */
1621         f = calloc(1, sizeof(struct extern_func));
1622         CHECK(func, ENOMEM);
1623
1624         /* Node initialization. */
1625         strcpy(f->name, name);
1626         f->mailbox_struct_type = mailbox_struct_type;
1627         f->func = func;
1628         f->struct_id = p->n_structs;
1629         f->id = p->n_extern_funcs;
1630
1631         /* Node add to tailq. */
1632         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1633         p->n_extern_funcs++;
1634         p->n_structs++;
1635
1636         return 0;
1637 }
1638
1639 static int
1640 extern_func_build(struct rte_swx_pipeline *p)
1641 {
1642         uint32_t i;
1643
1644         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1645                 struct thread *t = &p->threads[i];
1646                 struct extern_func *func;
1647
1648                 /* Memory allocation. */
1649                 t->extern_funcs = calloc(p->n_extern_funcs,
1650                                          sizeof(struct extern_func_runtime));
1651                 CHECK(t->extern_funcs, ENOMEM);
1652
1653                 /* Extern function. */
1654                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1655                         struct extern_func_runtime *r =
1656                                 &t->extern_funcs[func->id];
1657                         uint32_t mailbox_size =
1658                                 func->mailbox_struct_type->n_bits / 8;
1659
1660                         r->func = func->func;
1661
1662                         r->mailbox = calloc(1, mailbox_size);
1663                         CHECK(r->mailbox, ENOMEM);
1664
1665                         t->structs[func->struct_id] = r->mailbox;
1666                 }
1667         }
1668
1669         return 0;
1670 }
1671
1672 static void
1673 extern_func_build_free(struct rte_swx_pipeline *p)
1674 {
1675         uint32_t i;
1676
1677         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1678                 struct thread *t = &p->threads[i];
1679                 uint32_t j;
1680
1681                 if (!t->extern_funcs)
1682                         continue;
1683
1684                 for (j = 0; j < p->n_extern_funcs; j++) {
1685                         struct extern_func_runtime *r = &t->extern_funcs[j];
1686
1687                         free(r->mailbox);
1688                 }
1689
1690                 free(t->extern_funcs);
1691                 t->extern_funcs = NULL;
1692         }
1693 }
1694
1695 static void
1696 extern_func_free(struct rte_swx_pipeline *p)
1697 {
1698         extern_func_build_free(p);
1699
1700         for ( ; ; ) {
1701                 struct extern_func *elem;
1702
1703                 elem = TAILQ_FIRST(&p->extern_funcs);
1704                 if (!elem)
1705                         break;
1706
1707                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1708                 free(elem);
1709         }
1710 }
1711
1712 /*
1713  * Header.
1714  */
1715 static struct header *
1716 header_find(struct rte_swx_pipeline *p, const char *name)
1717 {
1718         struct header *elem;
1719
1720         TAILQ_FOREACH(elem, &p->headers, node)
1721                 if (strcmp(elem->name, name) == 0)
1722                         return elem;
1723
1724         return NULL;
1725 }
1726
1727 static struct header *
1728 header_parse(struct rte_swx_pipeline *p,
1729              const char *name)
1730 {
1731         if (name[0] != 'h' || name[1] != '.')
1732                 return NULL;
1733
1734         return header_find(p, &name[2]);
1735 }
1736
1737 static struct field *
1738 header_field_parse(struct rte_swx_pipeline *p,
1739                    const char *name,
1740                    struct header **header)
1741 {
1742         struct header *h;
1743         struct field *f;
1744         char *header_name, *field_name;
1745
1746         if ((name[0] != 'h') || (name[1] != '.'))
1747                 return NULL;
1748
1749         header_name = strdup(&name[2]);
1750         if (!header_name)
1751                 return NULL;
1752
1753         field_name = strchr(header_name, '.');
1754         if (!field_name) {
1755                 free(header_name);
1756                 return NULL;
1757         }
1758
1759         *field_name = 0;
1760         field_name++;
1761
1762         h = header_find(p, header_name);
1763         if (!h) {
1764                 free(header_name);
1765                 return NULL;
1766         }
1767
1768         f = struct_type_field_find(h->st, field_name);
1769         if (!f) {
1770                 free(header_name);
1771                 return NULL;
1772         }
1773
1774         if (header)
1775                 *header = h;
1776
1777         free(header_name);
1778         return f;
1779 }
1780
1781 int
1782 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1783                                         const char *name,
1784                                         const char *struct_type_name)
1785 {
1786         struct struct_type *st;
1787         struct header *h;
1788         size_t n_headers_max;
1789
1790         CHECK(p, EINVAL);
1791         CHECK_NAME(name, EINVAL);
1792         CHECK_NAME(struct_type_name, EINVAL);
1793
1794         CHECK(!header_find(p, name), EEXIST);
1795
1796         st = struct_type_find(p, struct_type_name);
1797         CHECK(st, EINVAL);
1798
1799         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1800         CHECK(p->n_headers < n_headers_max, ENOSPC);
1801
1802         /* Node allocation. */
1803         h = calloc(1, sizeof(struct header));
1804         CHECK(h, ENOMEM);
1805
1806         /* Node initialization. */
1807         strcpy(h->name, name);
1808         h->st = st;
1809         h->struct_id = p->n_structs;
1810         h->id = p->n_headers;
1811
1812         /* Node add to tailq. */
1813         TAILQ_INSERT_TAIL(&p->headers, h, node);
1814         p->n_headers++;
1815         p->n_structs++;
1816
1817         return 0;
1818 }
1819
1820 static int
1821 header_build(struct rte_swx_pipeline *p)
1822 {
1823         struct header *h;
1824         uint32_t n_bytes = 0, i;
1825
1826         TAILQ_FOREACH(h, &p->headers, node) {
1827                 n_bytes += h->st->n_bits / 8;
1828         }
1829
1830         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1831                 struct thread *t = &p->threads[i];
1832                 uint32_t offset = 0;
1833
1834                 t->headers = calloc(p->n_headers,
1835                                     sizeof(struct header_runtime));
1836                 CHECK(t->headers, ENOMEM);
1837
1838                 t->headers_out = calloc(p->n_headers,
1839                                         sizeof(struct header_out_runtime));
1840                 CHECK(t->headers_out, ENOMEM);
1841
1842                 t->header_storage = calloc(1, n_bytes);
1843                 CHECK(t->header_storage, ENOMEM);
1844
1845                 t->header_out_storage = calloc(1, n_bytes);
1846                 CHECK(t->header_out_storage, ENOMEM);
1847
1848                 TAILQ_FOREACH(h, &p->headers, node) {
1849                         uint8_t *header_storage;
1850
1851                         header_storage = &t->header_storage[offset];
1852                         offset += h->st->n_bits / 8;
1853
1854                         t->headers[h->id].ptr0 = header_storage;
1855                         t->structs[h->struct_id] = header_storage;
1856                 }
1857         }
1858
1859         return 0;
1860 }
1861
1862 static void
1863 header_build_free(struct rte_swx_pipeline *p)
1864 {
1865         uint32_t i;
1866
1867         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1868                 struct thread *t = &p->threads[i];
1869
1870                 free(t->headers_out);
1871                 t->headers_out = NULL;
1872
1873                 free(t->headers);
1874                 t->headers = NULL;
1875
1876                 free(t->header_out_storage);
1877                 t->header_out_storage = NULL;
1878
1879                 free(t->header_storage);
1880                 t->header_storage = NULL;
1881         }
1882 }
1883
1884 static void
1885 header_free(struct rte_swx_pipeline *p)
1886 {
1887         header_build_free(p);
1888
1889         for ( ; ; ) {
1890                 struct header *elem;
1891
1892                 elem = TAILQ_FIRST(&p->headers);
1893                 if (!elem)
1894                         break;
1895
1896                 TAILQ_REMOVE(&p->headers, elem, node);
1897                 free(elem);
1898         }
1899 }
1900
1901 /*
1902  * Meta-data.
1903  */
1904 static struct field *
1905 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1906 {
1907         if (!p->metadata_st)
1908                 return NULL;
1909
1910         if (name[0] != 'm' || name[1] != '.')
1911                 return NULL;
1912
1913         return struct_type_field_find(p->metadata_st, &name[2]);
1914 }
1915
1916 int
1917 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1918                                           const char *struct_type_name)
1919 {
1920         struct struct_type *st = NULL;
1921
1922         CHECK(p, EINVAL);
1923
1924         CHECK_NAME(struct_type_name, EINVAL);
1925         st  = struct_type_find(p, struct_type_name);
1926         CHECK(st, EINVAL);
1927         CHECK(!p->metadata_st, EINVAL);
1928
1929         p->metadata_st = st;
1930         p->metadata_struct_id = p->n_structs;
1931
1932         p->n_structs++;
1933
1934         return 0;
1935 }
1936
1937 static int
1938 metadata_build(struct rte_swx_pipeline *p)
1939 {
1940         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1941         uint32_t i;
1942
1943         /* Thread-level initialization. */
1944         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1945                 struct thread *t = &p->threads[i];
1946                 uint8_t *metadata;
1947
1948                 metadata = calloc(1, n_bytes);
1949                 CHECK(metadata, ENOMEM);
1950
1951                 t->metadata = metadata;
1952                 t->structs[p->metadata_struct_id] = metadata;
1953         }
1954
1955         return 0;
1956 }
1957
1958 static void
1959 metadata_build_free(struct rte_swx_pipeline *p)
1960 {
1961         uint32_t i;
1962
1963         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1964                 struct thread *t = &p->threads[i];
1965
1966                 free(t->metadata);
1967                 t->metadata = NULL;
1968         }
1969 }
1970
1971 static void
1972 metadata_free(struct rte_swx_pipeline *p)
1973 {
1974         metadata_build_free(p);
1975 }
1976
1977 /*
1978  * Instruction.
1979  */
1980 static struct field *
1981 action_field_parse(struct action *action, const char *name);
1982
1983 static struct field *
1984 struct_field_parse(struct rte_swx_pipeline *p,
1985                    struct action *action,
1986                    const char *name,
1987                    uint32_t *struct_id)
1988 {
1989         struct field *f;
1990
1991         switch (name[0]) {
1992         case 'h':
1993         {
1994                 struct header *header;
1995
1996                 f = header_field_parse(p, name, &header);
1997                 if (!f)
1998                         return NULL;
1999
2000                 *struct_id = header->struct_id;
2001                 return f;
2002         }
2003
2004         case 'm':
2005         {
2006                 f = metadata_field_parse(p, name);
2007                 if (!f)
2008                         return NULL;
2009
2010                 *struct_id = p->metadata_struct_id;
2011                 return f;
2012         }
2013
2014         case 't':
2015         {
2016                 if (!action)
2017                         return NULL;
2018
2019                 f = action_field_parse(action, name);
2020                 if (!f)
2021                         return NULL;
2022
2023                 *struct_id = 0;
2024                 return f;
2025         }
2026
2027         case 'e':
2028         {
2029                 struct extern_obj *obj;
2030
2031                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2032                 if (!f)
2033                         return NULL;
2034
2035                 *struct_id = obj->struct_id;
2036                 return f;
2037         }
2038
2039         case 'f':
2040         {
2041                 struct extern_func *func;
2042
2043                 f = extern_func_mailbox_field_parse(p, name, &func);
2044                 if (!f)
2045                         return NULL;
2046
2047                 *struct_id = func->struct_id;
2048                 return f;
2049         }
2050
2051         default:
2052                 return NULL;
2053         }
2054 }
2055
2056 static inline void
2057 pipeline_port_inc(struct rte_swx_pipeline *p)
2058 {
2059         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2060 }
2061
2062 static inline void
2063 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2064 {
2065         t->ip = p->instructions;
2066 }
2067
2068 static inline void
2069 thread_ip_action_call(struct rte_swx_pipeline *p,
2070                       struct thread *t,
2071                       uint32_t action_id)
2072 {
2073         t->ret = t->ip + 1;
2074         t->ip = p->action_instructions[action_id];
2075 }
2076
2077 static inline void
2078 thread_ip_inc(struct rte_swx_pipeline *p);
2079
2080 static inline void
2081 thread_ip_inc(struct rte_swx_pipeline *p)
2082 {
2083         struct thread *t = &p->threads[p->thread_id];
2084
2085         t->ip++;
2086 }
2087
2088 static inline void
2089 thread_ip_inc_cond(struct thread *t, int cond)
2090 {
2091         t->ip += cond;
2092 }
2093
2094 static inline void
2095 thread_yield(struct rte_swx_pipeline *p)
2096 {
2097         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2098 }
2099
2100 /*
2101  * rx.
2102  */
2103 static int
2104 instr_rx_translate(struct rte_swx_pipeline *p,
2105                    struct action *action,
2106                    char **tokens,
2107                    int n_tokens,
2108                    struct instruction *instr,
2109                    struct instruction_data *data __rte_unused)
2110 {
2111         struct field *f;
2112
2113         CHECK(!action, EINVAL);
2114         CHECK(n_tokens == 2, EINVAL);
2115
2116         f = metadata_field_parse(p, tokens[1]);
2117         CHECK(f, EINVAL);
2118
2119         instr->type = INSTR_RX;
2120         instr->io.io.offset = f->offset / 8;
2121         instr->io.io.n_bits = f->n_bits;
2122         return 0;
2123 }
2124
2125 static inline void
2126 instr_rx_exec(struct rte_swx_pipeline *p);
2127
2128 static inline void
2129 instr_rx_exec(struct rte_swx_pipeline *p)
2130 {
2131         struct thread *t = &p->threads[p->thread_id];
2132         struct instruction *ip = t->ip;
2133         struct port_in_runtime *port = &p->in[p->port_id];
2134         struct rte_swx_pkt *pkt = &t->pkt;
2135         int pkt_received;
2136
2137         /* Packet. */
2138         pkt_received = port->pkt_rx(port->obj, pkt);
2139         t->ptr = &pkt->pkt[pkt->offset];
2140         rte_prefetch0(t->ptr);
2141
2142         TRACE("[Thread %2u] rx %s from port %u\n",
2143               p->thread_id,
2144               pkt_received ? "1 pkt" : "0 pkts",
2145               p->port_id);
2146
2147         /* Headers. */
2148         t->valid_headers = 0;
2149         t->n_headers_out = 0;
2150
2151         /* Meta-data. */
2152         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2153
2154         /* Tables. */
2155         t->table_state = p->table_state;
2156
2157         /* Thread. */
2158         pipeline_port_inc(p);
2159         thread_ip_inc_cond(t, pkt_received);
2160         thread_yield(p);
2161 }
2162
2163 /*
2164  * tx.
2165  */
2166 static int
2167 instr_tx_translate(struct rte_swx_pipeline *p,
2168                    struct action *action __rte_unused,
2169                    char **tokens,
2170                    int n_tokens,
2171                    struct instruction *instr,
2172                    struct instruction_data *data __rte_unused)
2173 {
2174         struct field *f;
2175
2176         CHECK(n_tokens == 2, EINVAL);
2177
2178         f = metadata_field_parse(p, tokens[1]);
2179         CHECK(f, EINVAL);
2180
2181         instr->type = INSTR_TX;
2182         instr->io.io.offset = f->offset / 8;
2183         instr->io.io.n_bits = f->n_bits;
2184         return 0;
2185 }
2186
2187 static inline void
2188 emit_handler(struct thread *t)
2189 {
2190         struct header_out_runtime *h0 = &t->headers_out[0];
2191         struct header_out_runtime *h1 = &t->headers_out[1];
2192         uint32_t offset = 0, i;
2193
2194         /* No header change or header decapsulation. */
2195         if ((t->n_headers_out == 1) &&
2196             (h0->ptr + h0->n_bytes == t->ptr)) {
2197                 TRACE("Emit handler: no header change or header decap.\n");
2198
2199                 t->pkt.offset -= h0->n_bytes;
2200                 t->pkt.length += h0->n_bytes;
2201
2202                 return;
2203         }
2204
2205         /* Header encapsulation (optionally, with prior header decasulation). */
2206         if ((t->n_headers_out == 2) &&
2207             (h1->ptr + h1->n_bytes == t->ptr) &&
2208             (h0->ptr == h0->ptr0)) {
2209                 uint32_t offset;
2210
2211                 TRACE("Emit handler: header encapsulation.\n");
2212
2213                 offset = h0->n_bytes + h1->n_bytes;
2214                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2215                 t->pkt.offset -= offset;
2216                 t->pkt.length += offset;
2217
2218                 return;
2219         }
2220
2221         /* Header insertion. */
2222         /* TBD */
2223
2224         /* Header extraction. */
2225         /* TBD */
2226
2227         /* For any other case. */
2228         TRACE("Emit handler: complex case.\n");
2229
2230         for (i = 0; i < t->n_headers_out; i++) {
2231                 struct header_out_runtime *h = &t->headers_out[i];
2232
2233                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2234                 offset += h->n_bytes;
2235         }
2236
2237         if (offset) {
2238                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2239                 t->pkt.offset -= offset;
2240                 t->pkt.length += offset;
2241         }
2242 }
2243
2244 static inline void
2245 instr_tx_exec(struct rte_swx_pipeline *p);
2246
2247 static inline void
2248 instr_tx_exec(struct rte_swx_pipeline *p)
2249 {
2250         struct thread *t = &p->threads[p->thread_id];
2251         struct instruction *ip = t->ip;
2252         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2253         struct port_out_runtime *port = &p->out[port_id];
2254         struct rte_swx_pkt *pkt = &t->pkt;
2255
2256         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2257               p->thread_id,
2258               (uint32_t)port_id);
2259
2260         /* Headers. */
2261         emit_handler(t);
2262
2263         /* Packet. */
2264         port->pkt_tx(port->obj, pkt);
2265
2266         /* Thread. */
2267         thread_ip_reset(p, t);
2268         instr_rx_exec(p);
2269 }
2270
2271 /*
2272  * extract.
2273  */
2274 static int
2275 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2276                             struct action *action,
2277                             char **tokens,
2278                             int n_tokens,
2279                             struct instruction *instr,
2280                             struct instruction_data *data __rte_unused)
2281 {
2282         struct header *h;
2283
2284         CHECK(!action, EINVAL);
2285         CHECK(n_tokens == 2, EINVAL);
2286
2287         h = header_parse(p, tokens[1]);
2288         CHECK(h, EINVAL);
2289
2290         instr->type = INSTR_HDR_EXTRACT;
2291         instr->io.hdr.header_id[0] = h->id;
2292         instr->io.hdr.struct_id[0] = h->struct_id;
2293         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2294         return 0;
2295 }
2296
2297 static inline void
2298 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2299
2300 static inline void
2301 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2302 {
2303         struct thread *t = &p->threads[p->thread_id];
2304         struct instruction *ip = t->ip;
2305         uint64_t valid_headers = t->valid_headers;
2306         uint8_t *ptr = t->ptr;
2307         uint32_t offset = t->pkt.offset;
2308         uint32_t length = t->pkt.length;
2309         uint32_t i;
2310
2311         for (i = 0; i < n_extract; i++) {
2312                 uint32_t header_id = ip->io.hdr.header_id[i];
2313                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2314                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2315
2316                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2317                       p->thread_id,
2318                       header_id,
2319                       n_bytes);
2320
2321                 /* Headers. */
2322                 t->structs[struct_id] = ptr;
2323                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2324
2325                 /* Packet. */
2326                 offset += n_bytes;
2327                 length -= n_bytes;
2328                 ptr += n_bytes;
2329         }
2330
2331         /* Headers. */
2332         t->valid_headers = valid_headers;
2333
2334         /* Packet. */
2335         t->pkt.offset = offset;
2336         t->pkt.length = length;
2337         t->ptr = ptr;
2338 }
2339
2340 static inline void
2341 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2342 {
2343         __instr_hdr_extract_exec(p, 1);
2344
2345         /* Thread. */
2346         thread_ip_inc(p);
2347 }
2348
2349 static inline void
2350 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2351 {
2352         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2353               p->thread_id);
2354
2355         __instr_hdr_extract_exec(p, 2);
2356
2357         /* Thread. */
2358         thread_ip_inc(p);
2359 }
2360
2361 static inline void
2362 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2363 {
2364         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2365               p->thread_id);
2366
2367         __instr_hdr_extract_exec(p, 3);
2368
2369         /* Thread. */
2370         thread_ip_inc(p);
2371 }
2372
2373 static inline void
2374 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2375 {
2376         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2377               p->thread_id);
2378
2379         __instr_hdr_extract_exec(p, 4);
2380
2381         /* Thread. */
2382         thread_ip_inc(p);
2383 }
2384
2385 static inline void
2386 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2387 {
2388         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2389               p->thread_id);
2390
2391         __instr_hdr_extract_exec(p, 5);
2392
2393         /* Thread. */
2394         thread_ip_inc(p);
2395 }
2396
2397 static inline void
2398 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2399 {
2400         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2401               p->thread_id);
2402
2403         __instr_hdr_extract_exec(p, 6);
2404
2405         /* Thread. */
2406         thread_ip_inc(p);
2407 }
2408
2409 static inline void
2410 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2411 {
2412         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2413               p->thread_id);
2414
2415         __instr_hdr_extract_exec(p, 7);
2416
2417         /* Thread. */
2418         thread_ip_inc(p);
2419 }
2420
2421 static inline void
2422 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2423 {
2424         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2425               p->thread_id);
2426
2427         __instr_hdr_extract_exec(p, 8);
2428
2429         /* Thread. */
2430         thread_ip_inc(p);
2431 }
2432
2433 /*
2434  * emit.
2435  */
2436 static int
2437 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2438                          struct action *action __rte_unused,
2439                          char **tokens,
2440                          int n_tokens,
2441                          struct instruction *instr,
2442                          struct instruction_data *data __rte_unused)
2443 {
2444         struct header *h;
2445
2446         CHECK(n_tokens == 2, EINVAL);
2447
2448         h = header_parse(p, tokens[1]);
2449         CHECK(h, EINVAL);
2450
2451         instr->type = INSTR_HDR_EMIT;
2452         instr->io.hdr.header_id[0] = h->id;
2453         instr->io.hdr.struct_id[0] = h->struct_id;
2454         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2455         return 0;
2456 }
2457
2458 static inline void
2459 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
2460
2461 static inline void
2462 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
2463 {
2464         struct thread *t = &p->threads[p->thread_id];
2465         struct instruction *ip = t->ip;
2466         uint32_t n_headers_out = t->n_headers_out;
2467         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
2468         uint8_t *ho_ptr = NULL;
2469         uint32_t ho_nbytes = 0, i;
2470
2471         for (i = 0; i < n_emit; i++) {
2472                 uint32_t header_id = ip->io.hdr.header_id[i];
2473                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2474                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2475
2476                 struct header_runtime *hi = &t->headers[header_id];
2477                 uint8_t *hi_ptr = t->structs[struct_id];
2478
2479                 TRACE("[Thread %2u]: emit header %u\n",
2480                       p->thread_id,
2481                       header_id);
2482
2483                 /* Headers. */
2484                 if (!i) {
2485                         if (!t->n_headers_out) {
2486                                 ho = &t->headers_out[0];
2487
2488                                 ho->ptr0 = hi->ptr0;
2489                                 ho->ptr = hi_ptr;
2490
2491                                 ho_ptr = hi_ptr;
2492                                 ho_nbytes = n_bytes;
2493
2494                                 n_headers_out = 1;
2495
2496                                 continue;
2497                         } else {
2498                                 ho_ptr = ho->ptr;
2499                                 ho_nbytes = ho->n_bytes;
2500                         }
2501                 }
2502
2503                 if (ho_ptr + ho_nbytes == hi_ptr) {
2504                         ho_nbytes += n_bytes;
2505                 } else {
2506                         ho->n_bytes = ho_nbytes;
2507
2508                         ho++;
2509                         ho->ptr0 = hi->ptr0;
2510                         ho->ptr = hi_ptr;
2511
2512                         ho_ptr = hi_ptr;
2513                         ho_nbytes = n_bytes;
2514
2515                         n_headers_out++;
2516                 }
2517         }
2518
2519         ho->n_bytes = ho_nbytes;
2520         t->n_headers_out = n_headers_out;
2521 }
2522
2523 static inline void
2524 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2525 {
2526         __instr_hdr_emit_exec(p, 1);
2527
2528         /* Thread. */
2529         thread_ip_inc(p);
2530 }
2531
2532 static inline void
2533 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2534 {
2535         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2536               p->thread_id);
2537
2538         __instr_hdr_emit_exec(p, 1);
2539         instr_tx_exec(p);
2540 }
2541
2542 static inline void
2543 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2544 {
2545         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2546               p->thread_id);
2547
2548         __instr_hdr_emit_exec(p, 2);
2549         instr_tx_exec(p);
2550 }
2551
2552 static inline void
2553 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2554 {
2555         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2556               p->thread_id);
2557
2558         __instr_hdr_emit_exec(p, 3);
2559         instr_tx_exec(p);
2560 }
2561
2562 static inline void
2563 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2564 {
2565         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2566               p->thread_id);
2567
2568         __instr_hdr_emit_exec(p, 4);
2569         instr_tx_exec(p);
2570 }
2571
2572 static inline void
2573 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2574 {
2575         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2576               p->thread_id);
2577
2578         __instr_hdr_emit_exec(p, 5);
2579         instr_tx_exec(p);
2580 }
2581
2582 static inline void
2583 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2584 {
2585         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2586               p->thread_id);
2587
2588         __instr_hdr_emit_exec(p, 6);
2589         instr_tx_exec(p);
2590 }
2591
2592 static inline void
2593 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2594 {
2595         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2596               p->thread_id);
2597
2598         __instr_hdr_emit_exec(p, 7);
2599         instr_tx_exec(p);
2600 }
2601
2602 static inline void
2603 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2604 {
2605         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2606               p->thread_id);
2607
2608         __instr_hdr_emit_exec(p, 8);
2609         instr_tx_exec(p);
2610 }
2611
2612 /*
2613  * validate.
2614  */
2615 static int
2616 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2617                              struct action *action __rte_unused,
2618                              char **tokens,
2619                              int n_tokens,
2620                              struct instruction *instr,
2621                              struct instruction_data *data __rte_unused)
2622 {
2623         struct header *h;
2624
2625         CHECK(n_tokens == 2, EINVAL);
2626
2627         h = header_parse(p, tokens[1]);
2628         CHECK(h, EINVAL);
2629
2630         instr->type = INSTR_HDR_VALIDATE;
2631         instr->valid.header_id = h->id;
2632         return 0;
2633 }
2634
2635 static inline void
2636 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2637 {
2638         struct thread *t = &p->threads[p->thread_id];
2639         struct instruction *ip = t->ip;
2640         uint32_t header_id = ip->valid.header_id;
2641
2642         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2643
2644         /* Headers. */
2645         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2646
2647         /* Thread. */
2648         thread_ip_inc(p);
2649 }
2650
2651 /*
2652  * invalidate.
2653  */
2654 static int
2655 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2656                                struct action *action __rte_unused,
2657                                char **tokens,
2658                                int n_tokens,
2659                                struct instruction *instr,
2660                                struct instruction_data *data __rte_unused)
2661 {
2662         struct header *h;
2663
2664         CHECK(n_tokens == 2, EINVAL);
2665
2666         h = header_parse(p, tokens[1]);
2667         CHECK(h, EINVAL);
2668
2669         instr->type = INSTR_HDR_INVALIDATE;
2670         instr->valid.header_id = h->id;
2671         return 0;
2672 }
2673
2674 static inline void
2675 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2676 {
2677         struct thread *t = &p->threads[p->thread_id];
2678         struct instruction *ip = t->ip;
2679         uint32_t header_id = ip->valid.header_id;
2680
2681         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2682
2683         /* Headers. */
2684         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2685
2686         /* Thread. */
2687         thread_ip_inc(p);
2688 }
2689
2690 /*
2691  * table.
2692  */
2693 static struct table *
2694 table_find(struct rte_swx_pipeline *p, const char *name);
2695
2696 static int
2697 instr_table_translate(struct rte_swx_pipeline *p,
2698                       struct action *action,
2699                       char **tokens,
2700                       int n_tokens,
2701                       struct instruction *instr,
2702                       struct instruction_data *data __rte_unused)
2703 {
2704         struct table *t;
2705
2706         CHECK(!action, EINVAL);
2707         CHECK(n_tokens == 2, EINVAL);
2708
2709         t = table_find(p, tokens[1]);
2710         CHECK(t, EINVAL);
2711
2712         instr->type = INSTR_TABLE;
2713         instr->table.table_id = t->id;
2714         return 0;
2715 }
2716
2717 static inline void
2718 instr_table_exec(struct rte_swx_pipeline *p)
2719 {
2720         struct thread *t = &p->threads[p->thread_id];
2721         struct instruction *ip = t->ip;
2722         uint32_t table_id = ip->table.table_id;
2723         struct rte_swx_table_state *ts = &t->table_state[table_id];
2724         struct table_runtime *table = &t->tables[table_id];
2725         uint64_t action_id;
2726         uint8_t *action_data;
2727         int done, hit;
2728
2729         /* Table. */
2730         done = table->func(ts->obj,
2731                            table->mailbox,
2732                            table->key,
2733                            &action_id,
2734                            &action_data,
2735                            &hit);
2736         if (!done) {
2737                 /* Thread. */
2738                 TRACE("[Thread %2u] table %u (not finalized)\n",
2739                       p->thread_id,
2740                       table_id);
2741
2742                 thread_yield(p);
2743                 return;
2744         }
2745
2746         action_id = hit ? action_id : ts->default_action_id;
2747         action_data = hit ? action_data : ts->default_action_data;
2748
2749         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2750               p->thread_id,
2751               table_id,
2752               hit ? "hit" : "miss",
2753               (uint32_t)action_id);
2754
2755         t->action_id = action_id;
2756         t->structs[0] = action_data;
2757         t->hit = hit;
2758
2759         /* Thread. */
2760         thread_ip_action_call(p, t, action_id);
2761 }
2762
2763 /*
2764  * mov.
2765  */
2766 static int
2767 instr_mov_translate(struct rte_swx_pipeline *p,
2768                     struct action *action,
2769                     char **tokens,
2770                     int n_tokens,
2771                     struct instruction *instr,
2772                     struct instruction_data *data __rte_unused)
2773 {
2774         char *dst = tokens[1], *src = tokens[2];
2775         struct field *fdst, *fsrc;
2776         uint32_t dst_struct_id, src_struct_id, src_val;
2777
2778         CHECK(n_tokens == 3, EINVAL);
2779
2780         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2781         CHECK(fdst, EINVAL);
2782
2783         /* MOV or MOV_S. */
2784         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2785         if (fsrc) {
2786                 instr->type = INSTR_MOV;
2787                 if ((dst[0] == 'h' && src[0] != 'h') ||
2788                     (dst[0] != 'h' && src[0] == 'h'))
2789                         instr->type = INSTR_MOV_S;
2790
2791                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2792                 instr->mov.dst.n_bits = fdst->n_bits;
2793                 instr->mov.dst.offset = fdst->offset / 8;
2794                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2795                 instr->mov.src.n_bits = fsrc->n_bits;
2796                 instr->mov.src.offset = fsrc->offset / 8;
2797                 return 0;
2798         }
2799
2800         /* MOV_I. */
2801         src_val = strtoul(src, &src, 0);
2802         CHECK(!src[0], EINVAL);
2803
2804         if (dst[0] == 'h')
2805                 src_val = htonl(src_val);
2806
2807         instr->type = INSTR_MOV_I;
2808         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2809         instr->mov.dst.n_bits = fdst->n_bits;
2810         instr->mov.dst.offset = fdst->offset / 8;
2811         instr->mov.src_val = (uint32_t)src_val;
2812         return 0;
2813 }
2814
2815 static inline void
2816 instr_mov_exec(struct rte_swx_pipeline *p)
2817 {
2818         struct thread *t = &p->threads[p->thread_id];
2819         struct instruction *ip = t->ip;
2820
2821         TRACE("[Thread %2u] mov\n",
2822               p->thread_id);
2823
2824         MOV(t, ip);
2825
2826         /* Thread. */
2827         thread_ip_inc(p);
2828 }
2829
2830 static inline void
2831 instr_mov_s_exec(struct rte_swx_pipeline *p)
2832 {
2833         struct thread *t = &p->threads[p->thread_id];
2834         struct instruction *ip = t->ip;
2835
2836         TRACE("[Thread %2u] mov (s)\n",
2837               p->thread_id);
2838
2839         MOV_S(t, ip);
2840
2841         /* Thread. */
2842         thread_ip_inc(p);
2843 }
2844
2845 static inline void
2846 instr_mov_i_exec(struct rte_swx_pipeline *p)
2847 {
2848         struct thread *t = &p->threads[p->thread_id];
2849         struct instruction *ip = t->ip;
2850
2851         TRACE("[Thread %2u] mov m.f %x\n",
2852               p->thread_id,
2853               ip->mov.src_val);
2854
2855         MOV_I(t, ip);
2856
2857         /* Thread. */
2858         thread_ip_inc(p);
2859 }
2860
2861 /*
2862  * dma.
2863  */
2864 static int
2865 instr_dma_translate(struct rte_swx_pipeline *p,
2866                     struct action *action,
2867                     char **tokens,
2868                     int n_tokens,
2869                     struct instruction *instr,
2870                     struct instruction_data *data __rte_unused)
2871 {
2872         char *dst = tokens[1];
2873         char *src = tokens[2];
2874         struct header *h;
2875         struct field *tf;
2876
2877         CHECK(action, EINVAL);
2878         CHECK(n_tokens == 3, EINVAL);
2879
2880         h = header_parse(p, dst);
2881         CHECK(h, EINVAL);
2882
2883         tf = action_field_parse(action, src);
2884         CHECK(tf, EINVAL);
2885
2886         instr->type = INSTR_DMA_HT;
2887         instr->dma.dst.header_id[0] = h->id;
2888         instr->dma.dst.struct_id[0] = h->struct_id;
2889         instr->dma.n_bytes[0] = h->st->n_bits / 8;
2890         instr->dma.src.offset[0] = tf->offset / 8;
2891
2892         return 0;
2893 }
2894
2895 static inline void
2896 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2897
2898 static inline void
2899 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2900 {
2901         struct thread *t = &p->threads[p->thread_id];
2902         struct instruction *ip = t->ip;
2903         uint8_t *action_data = t->structs[0];
2904         uint64_t valid_headers = t->valid_headers;
2905         uint32_t i;
2906
2907         for (i = 0; i < n_dma; i++) {
2908                 uint32_t header_id = ip->dma.dst.header_id[i];
2909                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2910                 uint32_t offset = ip->dma.src.offset[i];
2911                 uint32_t n_bytes = ip->dma.n_bytes[i];
2912
2913                 struct header_runtime *h = &t->headers[header_id];
2914                 uint8_t *h_ptr0 = h->ptr0;
2915                 uint8_t *h_ptr = t->structs[struct_id];
2916
2917                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2918                         h_ptr : h_ptr0;
2919                 void *src = &action_data[offset];
2920
2921                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2922
2923                 /* Headers. */
2924                 memcpy(dst, src, n_bytes);
2925                 t->structs[struct_id] = dst;
2926                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2927         }
2928
2929         t->valid_headers = valid_headers;
2930 }
2931
2932 static inline void
2933 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2934 {
2935         __instr_dma_ht_exec(p, 1);
2936
2937         /* Thread. */
2938         thread_ip_inc(p);
2939 }
2940
2941 static inline void
2942 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2943 {
2944         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2945               p->thread_id);
2946
2947         __instr_dma_ht_exec(p, 2);
2948
2949         /* Thread. */
2950         thread_ip_inc(p);
2951 }
2952
2953 static inline void
2954 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2955 {
2956         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2957               p->thread_id);
2958
2959         __instr_dma_ht_exec(p, 3);
2960
2961         /* Thread. */
2962         thread_ip_inc(p);
2963 }
2964
2965 static inline void
2966 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2967 {
2968         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2969               p->thread_id);
2970
2971         __instr_dma_ht_exec(p, 4);
2972
2973         /* Thread. */
2974         thread_ip_inc(p);
2975 }
2976
2977 static inline void
2978 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2979 {
2980         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2981               p->thread_id);
2982
2983         __instr_dma_ht_exec(p, 5);
2984
2985         /* Thread. */
2986         thread_ip_inc(p);
2987 }
2988
2989 static inline void
2990 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2991 {
2992         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2993               p->thread_id);
2994
2995         __instr_dma_ht_exec(p, 6);
2996
2997         /* Thread. */
2998         thread_ip_inc(p);
2999 }
3000
3001 static inline void
3002 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3003 {
3004         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3005               p->thread_id);
3006
3007         __instr_dma_ht_exec(p, 7);
3008
3009         /* Thread. */
3010         thread_ip_inc(p);
3011 }
3012
3013 static inline void
3014 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3015 {
3016         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3017               p->thread_id);
3018
3019         __instr_dma_ht_exec(p, 8);
3020
3021         /* Thread. */
3022         thread_ip_inc(p);
3023 }
3024
3025 /*
3026  * alu.
3027  */
3028 static int
3029 instr_alu_add_translate(struct rte_swx_pipeline *p,
3030                         struct action *action,
3031                         char **tokens,
3032                         int n_tokens,
3033                         struct instruction *instr,
3034                         struct instruction_data *data __rte_unused)
3035 {
3036         char *dst = tokens[1], *src = tokens[2];
3037         struct field *fdst, *fsrc;
3038         uint32_t dst_struct_id, src_struct_id, src_val;
3039
3040         CHECK(n_tokens == 3, EINVAL);
3041
3042         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3043         CHECK(fdst, EINVAL);
3044
3045         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3046         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3047         if (fsrc) {
3048                 instr->type = INSTR_ALU_ADD;
3049                 if (dst[0] == 'h' && src[0] == 'm')
3050                         instr->type = INSTR_ALU_ADD_HM;
3051                 if (dst[0] == 'm' && src[0] == 'h')
3052                         instr->type = INSTR_ALU_ADD_MH;
3053                 if (dst[0] == 'h' && src[0] == 'h')
3054                         instr->type = INSTR_ALU_ADD_HH;
3055
3056                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3057                 instr->alu.dst.n_bits = fdst->n_bits;
3058                 instr->alu.dst.offset = fdst->offset / 8;
3059                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3060                 instr->alu.src.n_bits = fsrc->n_bits;
3061                 instr->alu.src.offset = fsrc->offset / 8;
3062                 return 0;
3063         }
3064
3065         /* ADD_MI, ADD_HI. */
3066         src_val = strtoul(src, &src, 0);
3067         CHECK(!src[0], EINVAL);
3068
3069         instr->type = INSTR_ALU_ADD_MI;
3070         if (dst[0] == 'h')
3071                 instr->type = INSTR_ALU_ADD_HI;
3072
3073         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3074         instr->alu.dst.n_bits = fdst->n_bits;
3075         instr->alu.dst.offset = fdst->offset / 8;
3076         instr->alu.src_val = (uint32_t)src_val;
3077         return 0;
3078 }
3079
3080 static int
3081 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3082                         struct action *action,
3083                         char **tokens,
3084                         int n_tokens,
3085                         struct instruction *instr,
3086                         struct instruction_data *data __rte_unused)
3087 {
3088         char *dst = tokens[1], *src = tokens[2];
3089         struct field *fdst, *fsrc;
3090         uint32_t dst_struct_id, src_struct_id, src_val;
3091
3092         CHECK(n_tokens == 3, EINVAL);
3093
3094         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3095         CHECK(fdst, EINVAL);
3096
3097         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3098         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3099         if (fsrc) {
3100                 instr->type = INSTR_ALU_SUB;
3101                 if (dst[0] == 'h' && src[0] == 'm')
3102                         instr->type = INSTR_ALU_SUB_HM;
3103                 if (dst[0] == 'm' && src[0] == 'h')
3104                         instr->type = INSTR_ALU_SUB_MH;
3105                 if (dst[0] == 'h' && src[0] == 'h')
3106                         instr->type = INSTR_ALU_SUB_HH;
3107
3108                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3109                 instr->alu.dst.n_bits = fdst->n_bits;
3110                 instr->alu.dst.offset = fdst->offset / 8;
3111                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3112                 instr->alu.src.n_bits = fsrc->n_bits;
3113                 instr->alu.src.offset = fsrc->offset / 8;
3114                 return 0;
3115         }
3116
3117         /* SUB_MI, SUB_HI. */
3118         src_val = strtoul(src, &src, 0);
3119         CHECK(!src[0], EINVAL);
3120
3121         instr->type = INSTR_ALU_SUB_MI;
3122         if (dst[0] == 'h')
3123                 instr->type = INSTR_ALU_SUB_HI;
3124
3125         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3126         instr->alu.dst.n_bits = fdst->n_bits;
3127         instr->alu.dst.offset = fdst->offset / 8;
3128         instr->alu.src_val = (uint32_t)src_val;
3129         return 0;
3130 }
3131
3132 static int
3133 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3134                           struct action *action __rte_unused,
3135                           char **tokens,
3136                           int n_tokens,
3137                           struct instruction *instr,
3138                           struct instruction_data *data __rte_unused)
3139 {
3140         char *dst = tokens[1], *src = tokens[2];
3141         struct header *hdst, *hsrc;
3142         struct field *fdst, *fsrc;
3143
3144         CHECK(n_tokens == 3, EINVAL);
3145
3146         fdst = header_field_parse(p, dst, &hdst);
3147         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3148
3149         /* CKADD_FIELD. */
3150         fsrc = header_field_parse(p, src, &hsrc);
3151         if (fsrc) {
3152                 instr->type = INSTR_ALU_CKADD_FIELD;
3153                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3154                 instr->alu.dst.n_bits = fdst->n_bits;
3155                 instr->alu.dst.offset = fdst->offset / 8;
3156                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3157                 instr->alu.src.n_bits = fsrc->n_bits;
3158                 instr->alu.src.offset = fsrc->offset / 8;
3159                 return 0;
3160         }
3161
3162         /* CKADD_STRUCT, CKADD_STRUCT20. */
3163         hsrc = header_parse(p, src);
3164         CHECK(hsrc, EINVAL);
3165
3166         instr->type = INSTR_ALU_CKADD_STRUCT;
3167         if ((hsrc->st->n_bits / 8) == 20)
3168                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3169
3170         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3171         instr->alu.dst.n_bits = fdst->n_bits;
3172         instr->alu.dst.offset = fdst->offset / 8;
3173         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3174         instr->alu.src.n_bits = hsrc->st->n_bits;
3175         instr->alu.src.offset = 0; /* Unused. */
3176         return 0;
3177 }
3178
3179 static int
3180 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3181                           struct action *action __rte_unused,
3182                           char **tokens,
3183                           int n_tokens,
3184                           struct instruction *instr,
3185                           struct instruction_data *data __rte_unused)
3186 {
3187         char *dst = tokens[1], *src = tokens[2];
3188         struct header *hdst, *hsrc;
3189         struct field *fdst, *fsrc;
3190
3191         CHECK(n_tokens == 3, EINVAL);
3192
3193         fdst = header_field_parse(p, dst, &hdst);
3194         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3195
3196         fsrc = header_field_parse(p, src, &hsrc);
3197         CHECK(fsrc, EINVAL);
3198
3199         instr->type = INSTR_ALU_CKSUB_FIELD;
3200         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3201         instr->alu.dst.n_bits = fdst->n_bits;
3202         instr->alu.dst.offset = fdst->offset / 8;
3203         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3204         instr->alu.src.n_bits = fsrc->n_bits;
3205         instr->alu.src.offset = fsrc->offset / 8;
3206         return 0;
3207 }
3208
3209 static int
3210 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3211                         struct action *action,
3212                         char **tokens,
3213                         int n_tokens,
3214                         struct instruction *instr,
3215                         struct instruction_data *data __rte_unused)
3216 {
3217         char *dst = tokens[1], *src = tokens[2];
3218         struct field *fdst, *fsrc;
3219         uint32_t dst_struct_id, src_struct_id, src_val;
3220
3221         CHECK(n_tokens == 3, EINVAL);
3222
3223         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3224         CHECK(fdst, EINVAL);
3225
3226         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3227         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3228         if (fsrc) {
3229                 instr->type = INSTR_ALU_SHL;
3230                 if (dst[0] == 'h' && src[0] == 'm')
3231                         instr->type = INSTR_ALU_SHL_HM;
3232                 if (dst[0] == 'm' && src[0] == 'h')
3233                         instr->type = INSTR_ALU_SHL_MH;
3234                 if (dst[0] == 'h' && src[0] == 'h')
3235                         instr->type = INSTR_ALU_SHL_HH;
3236
3237                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3238                 instr->alu.dst.n_bits = fdst->n_bits;
3239                 instr->alu.dst.offset = fdst->offset / 8;
3240                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3241                 instr->alu.src.n_bits = fsrc->n_bits;
3242                 instr->alu.src.offset = fsrc->offset / 8;
3243                 return 0;
3244         }
3245
3246         /* SHL_MI, SHL_HI. */
3247         src_val = strtoul(src, &src, 0);
3248         CHECK(!src[0], EINVAL);
3249
3250         instr->type = INSTR_ALU_SHL_MI;
3251         if (dst[0] == 'h')
3252                 instr->type = INSTR_ALU_SHL_HI;
3253
3254         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3255         instr->alu.dst.n_bits = fdst->n_bits;
3256         instr->alu.dst.offset = fdst->offset / 8;
3257         instr->alu.src_val = (uint32_t)src_val;
3258         return 0;
3259 }
3260
3261 static int
3262 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3263                         struct action *action,
3264                         char **tokens,
3265                         int n_tokens,
3266                         struct instruction *instr,
3267                         struct instruction_data *data __rte_unused)
3268 {
3269         char *dst = tokens[1], *src = tokens[2];
3270         struct field *fdst, *fsrc;
3271         uint32_t dst_struct_id, src_struct_id, src_val;
3272
3273         CHECK(n_tokens == 3, EINVAL);
3274
3275         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3276         CHECK(fdst, EINVAL);
3277
3278         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3279         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3280         if (fsrc) {
3281                 instr->type = INSTR_ALU_SHR;
3282                 if (dst[0] == 'h' && src[0] == 'm')
3283                         instr->type = INSTR_ALU_SHR_HM;
3284                 if (dst[0] == 'm' && src[0] == 'h')
3285                         instr->type = INSTR_ALU_SHR_MH;
3286                 if (dst[0] == 'h' && src[0] == 'h')
3287                         instr->type = INSTR_ALU_SHR_HH;
3288
3289                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3290                 instr->alu.dst.n_bits = fdst->n_bits;
3291                 instr->alu.dst.offset = fdst->offset / 8;
3292                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3293                 instr->alu.src.n_bits = fsrc->n_bits;
3294                 instr->alu.src.offset = fsrc->offset / 8;
3295                 return 0;
3296         }
3297
3298         /* SHR_MI, SHR_HI. */
3299         src_val = strtoul(src, &src, 0);
3300         CHECK(!src[0], EINVAL);
3301
3302         instr->type = INSTR_ALU_SHR_MI;
3303         if (dst[0] == 'h')
3304                 instr->type = INSTR_ALU_SHR_HI;
3305
3306         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3307         instr->alu.dst.n_bits = fdst->n_bits;
3308         instr->alu.dst.offset = fdst->offset / 8;
3309         instr->alu.src_val = (uint32_t)src_val;
3310         return 0;
3311 }
3312
3313 static int
3314 instr_alu_and_translate(struct rte_swx_pipeline *p,
3315                         struct action *action,
3316                         char **tokens,
3317                         int n_tokens,
3318                         struct instruction *instr,
3319                         struct instruction_data *data __rte_unused)
3320 {
3321         char *dst = tokens[1], *src = tokens[2];
3322         struct field *fdst, *fsrc;
3323         uint32_t dst_struct_id, src_struct_id, src_val;
3324
3325         CHECK(n_tokens == 3, EINVAL);
3326
3327         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3328         CHECK(fdst, EINVAL);
3329
3330         /* AND or AND_S. */
3331         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3332         if (fsrc) {
3333                 instr->type = INSTR_ALU_AND;
3334                 if ((dst[0] == 'h' && src[0] != 'h') ||
3335                     (dst[0] != 'h' && src[0] == 'h'))
3336                         instr->type = INSTR_ALU_AND_S;
3337
3338                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3339                 instr->alu.dst.n_bits = fdst->n_bits;
3340                 instr->alu.dst.offset = fdst->offset / 8;
3341                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3342                 instr->alu.src.n_bits = fsrc->n_bits;
3343                 instr->alu.src.offset = fsrc->offset / 8;
3344                 return 0;
3345         }
3346
3347         /* AND_I. */
3348         src_val = strtoul(src, &src, 0);
3349         CHECK(!src[0], EINVAL);
3350
3351         if (dst[0] == 'h')
3352                 src_val = htonl(src_val);
3353
3354         instr->type = INSTR_ALU_AND_I;
3355         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3356         instr->alu.dst.n_bits = fdst->n_bits;
3357         instr->alu.dst.offset = fdst->offset / 8;
3358         instr->alu.src_val = (uint32_t)src_val;
3359         return 0;
3360 }
3361
3362 static int
3363 instr_alu_or_translate(struct rte_swx_pipeline *p,
3364                        struct action *action,
3365                        char **tokens,
3366                        int n_tokens,
3367                        struct instruction *instr,
3368                        struct instruction_data *data __rte_unused)
3369 {
3370         char *dst = tokens[1], *src = tokens[2];
3371         struct field *fdst, *fsrc;
3372         uint32_t dst_struct_id, src_struct_id, src_val;
3373
3374         CHECK(n_tokens == 3, EINVAL);
3375
3376         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3377         CHECK(fdst, EINVAL);
3378
3379         /* OR or OR_S. */
3380         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3381         if (fsrc) {
3382                 instr->type = INSTR_ALU_OR;
3383                 if ((dst[0] == 'h' && src[0] != 'h') ||
3384                     (dst[0] != 'h' && src[0] == 'h'))
3385                         instr->type = INSTR_ALU_OR_S;
3386
3387                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3388                 instr->alu.dst.n_bits = fdst->n_bits;
3389                 instr->alu.dst.offset = fdst->offset / 8;
3390                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3391                 instr->alu.src.n_bits = fsrc->n_bits;
3392                 instr->alu.src.offset = fsrc->offset / 8;
3393                 return 0;
3394         }
3395
3396         /* OR_I. */
3397         src_val = strtoul(src, &src, 0);
3398         CHECK(!src[0], EINVAL);
3399
3400         if (dst[0] == 'h')
3401                 src_val = htonl(src_val);
3402
3403         instr->type = INSTR_ALU_OR_I;
3404         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3405         instr->alu.dst.n_bits = fdst->n_bits;
3406         instr->alu.dst.offset = fdst->offset / 8;
3407         instr->alu.src_val = (uint32_t)src_val;
3408         return 0;
3409 }
3410
3411 static int
3412 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3413                         struct action *action,
3414                         char **tokens,
3415                         int n_tokens,
3416                         struct instruction *instr,
3417                         struct instruction_data *data __rte_unused)
3418 {
3419         char *dst = tokens[1], *src = tokens[2];
3420         struct field *fdst, *fsrc;
3421         uint32_t dst_struct_id, src_struct_id, src_val;
3422
3423         CHECK(n_tokens == 3, EINVAL);
3424
3425         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3426         CHECK(fdst, EINVAL);
3427
3428         /* XOR or XOR_S. */
3429         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3430         if (fsrc) {
3431                 instr->type = INSTR_ALU_XOR;
3432                 if ((dst[0] == 'h' && src[0] != 'h') ||
3433                     (dst[0] != 'h' && src[0] == 'h'))
3434                         instr->type = INSTR_ALU_XOR_S;
3435
3436                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3437                 instr->alu.dst.n_bits = fdst->n_bits;
3438                 instr->alu.dst.offset = fdst->offset / 8;
3439                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3440                 instr->alu.src.n_bits = fsrc->n_bits;
3441                 instr->alu.src.offset = fsrc->offset / 8;
3442                 return 0;
3443         }
3444
3445         /* XOR_I. */
3446         src_val = strtoul(src, &src, 0);
3447         CHECK(!src[0], EINVAL);
3448
3449         if (dst[0] == 'h')
3450                 src_val = htonl(src_val);
3451
3452         instr->type = INSTR_ALU_XOR_I;
3453         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3454         instr->alu.dst.n_bits = fdst->n_bits;
3455         instr->alu.dst.offset = fdst->offset / 8;
3456         instr->alu.src_val = (uint32_t)src_val;
3457         return 0;
3458 }
3459
3460 static inline void
3461 instr_alu_add_exec(struct rte_swx_pipeline *p)
3462 {
3463         struct thread *t = &p->threads[p->thread_id];
3464         struct instruction *ip = t->ip;
3465
3466         TRACE("[Thread %2u] add\n", p->thread_id);
3467
3468         /* Structs. */
3469         ALU(t, ip, +);
3470
3471         /* Thread. */
3472         thread_ip_inc(p);
3473 }
3474
3475 static inline void
3476 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3477 {
3478         struct thread *t = &p->threads[p->thread_id];
3479         struct instruction *ip = t->ip;
3480
3481         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3482
3483         /* Structs. */
3484         ALU_MH(t, ip, +);
3485
3486         /* Thread. */
3487         thread_ip_inc(p);
3488 }
3489
3490 static inline void
3491 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3492 {
3493         struct thread *t = &p->threads[p->thread_id];
3494         struct instruction *ip = t->ip;
3495
3496         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3497
3498         /* Structs. */
3499         ALU_HM(t, ip, +);
3500
3501         /* Thread. */
3502         thread_ip_inc(p);
3503 }
3504
3505 static inline void
3506 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3507 {
3508         struct thread *t = &p->threads[p->thread_id];
3509         struct instruction *ip = t->ip;
3510
3511         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3512
3513         /* Structs. */
3514         ALU_HH(t, ip, +);
3515
3516         /* Thread. */
3517         thread_ip_inc(p);
3518 }
3519
3520 static inline void
3521 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3522 {
3523         struct thread *t = &p->threads[p->thread_id];
3524         struct instruction *ip = t->ip;
3525
3526         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3527
3528         /* Structs. */
3529         ALU_MI(t, ip, +);
3530
3531         /* Thread. */
3532         thread_ip_inc(p);
3533 }
3534
3535 static inline void
3536 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3537 {
3538         struct thread *t = &p->threads[p->thread_id];
3539         struct instruction *ip = t->ip;
3540
3541         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3542
3543         /* Structs. */
3544         ALU_HI(t, ip, +);
3545
3546         /* Thread. */
3547         thread_ip_inc(p);
3548 }
3549
3550 static inline void
3551 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3552 {
3553         struct thread *t = &p->threads[p->thread_id];
3554         struct instruction *ip = t->ip;
3555
3556         TRACE("[Thread %2u] sub\n", p->thread_id);
3557
3558         /* Structs. */
3559         ALU(t, ip, -);
3560
3561         /* Thread. */
3562         thread_ip_inc(p);
3563 }
3564
3565 static inline void
3566 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3567 {
3568         struct thread *t = &p->threads[p->thread_id];
3569         struct instruction *ip = t->ip;
3570
3571         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3572
3573         /* Structs. */
3574         ALU_MH(t, ip, -);
3575
3576         /* Thread. */
3577         thread_ip_inc(p);
3578 }
3579
3580 static inline void
3581 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3582 {
3583         struct thread *t = &p->threads[p->thread_id];
3584         struct instruction *ip = t->ip;
3585
3586         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3587
3588         /* Structs. */
3589         ALU_HM(t, ip, -);
3590
3591         /* Thread. */
3592         thread_ip_inc(p);
3593 }
3594
3595 static inline void
3596 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3597 {
3598         struct thread *t = &p->threads[p->thread_id];
3599         struct instruction *ip = t->ip;
3600
3601         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3602
3603         /* Structs. */
3604         ALU_HH(t, ip, -);
3605
3606         /* Thread. */
3607         thread_ip_inc(p);
3608 }
3609
3610 static inline void
3611 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3612 {
3613         struct thread *t = &p->threads[p->thread_id];
3614         struct instruction *ip = t->ip;
3615
3616         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3617
3618         /* Structs. */
3619         ALU_MI(t, ip, -);
3620
3621         /* Thread. */
3622         thread_ip_inc(p);
3623 }
3624
3625 static inline void
3626 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3627 {
3628         struct thread *t = &p->threads[p->thread_id];
3629         struct instruction *ip = t->ip;
3630
3631         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3632
3633         /* Structs. */
3634         ALU_HI(t, ip, -);
3635
3636         /* Thread. */
3637         thread_ip_inc(p);
3638 }
3639
3640 static inline void
3641 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3642 {
3643         struct thread *t = &p->threads[p->thread_id];
3644         struct instruction *ip = t->ip;
3645
3646         TRACE("[Thread %2u] shl\n", p->thread_id);
3647
3648         /* Structs. */
3649         ALU(t, ip, <<);
3650
3651         /* Thread. */
3652         thread_ip_inc(p);
3653 }
3654
3655 static inline void
3656 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3657 {
3658         struct thread *t = &p->threads[p->thread_id];
3659         struct instruction *ip = t->ip;
3660
3661         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
3662
3663         /* Structs. */
3664         ALU_MH(t, ip, <<);
3665
3666         /* Thread. */
3667         thread_ip_inc(p);
3668 }
3669
3670 static inline void
3671 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3672 {
3673         struct thread *t = &p->threads[p->thread_id];
3674         struct instruction *ip = t->ip;
3675
3676         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
3677
3678         /* Structs. */
3679         ALU_HM(t, ip, <<);
3680
3681         /* Thread. */
3682         thread_ip_inc(p);
3683 }
3684
3685 static inline void
3686 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3687 {
3688         struct thread *t = &p->threads[p->thread_id];
3689         struct instruction *ip = t->ip;
3690
3691         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
3692
3693         /* Structs. */
3694         ALU_HH(t, ip, <<);
3695
3696         /* Thread. */
3697         thread_ip_inc(p);
3698 }
3699
3700 static inline void
3701 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3702 {
3703         struct thread *t = &p->threads[p->thread_id];
3704         struct instruction *ip = t->ip;
3705
3706         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
3707
3708         /* Structs. */
3709         ALU_MI(t, ip, <<);
3710
3711         /* Thread. */
3712         thread_ip_inc(p);
3713 }
3714
3715 static inline void
3716 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3717 {
3718         struct thread *t = &p->threads[p->thread_id];
3719         struct instruction *ip = t->ip;
3720
3721         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
3722
3723         /* Structs. */
3724         ALU_HI(t, ip, <<);
3725
3726         /* Thread. */
3727         thread_ip_inc(p);
3728 }
3729
3730 static inline void
3731 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3732 {
3733         struct thread *t = &p->threads[p->thread_id];
3734         struct instruction *ip = t->ip;
3735
3736         TRACE("[Thread %2u] shr\n", p->thread_id);
3737
3738         /* Structs. */
3739         ALU(t, ip, >>);
3740
3741         /* Thread. */
3742         thread_ip_inc(p);
3743 }
3744
3745 static inline void
3746 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3747 {
3748         struct thread *t = &p->threads[p->thread_id];
3749         struct instruction *ip = t->ip;
3750
3751         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
3752
3753         /* Structs. */
3754         ALU_MH(t, ip, >>);
3755
3756         /* Thread. */
3757         thread_ip_inc(p);
3758 }
3759
3760 static inline void
3761 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3762 {
3763         struct thread *t = &p->threads[p->thread_id];
3764         struct instruction *ip = t->ip;
3765
3766         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
3767
3768         /* Structs. */
3769         ALU_HM(t, ip, >>);
3770
3771         /* Thread. */
3772         thread_ip_inc(p);
3773 }
3774
3775 static inline void
3776 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3777 {
3778         struct thread *t = &p->threads[p->thread_id];
3779         struct instruction *ip = t->ip;
3780
3781         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
3782
3783         /* Structs. */
3784         ALU_HH(t, ip, >>);
3785
3786         /* Thread. */
3787         thread_ip_inc(p);
3788 }
3789
3790 static inline void
3791 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3792 {
3793         struct thread *t = &p->threads[p->thread_id];
3794         struct instruction *ip = t->ip;
3795
3796         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
3797
3798         /* Structs. */
3799         ALU_MI(t, ip, >>);
3800
3801         /* Thread. */
3802         thread_ip_inc(p);
3803 }
3804
3805 static inline void
3806 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3807 {
3808         struct thread *t = &p->threads[p->thread_id];
3809         struct instruction *ip = t->ip;
3810
3811         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
3812
3813         /* Structs. */
3814         ALU_HI(t, ip, >>);
3815
3816         /* Thread. */
3817         thread_ip_inc(p);
3818 }
3819
3820 static inline void
3821 instr_alu_and_exec(struct rte_swx_pipeline *p)
3822 {
3823         struct thread *t = &p->threads[p->thread_id];
3824         struct instruction *ip = t->ip;
3825
3826         TRACE("[Thread %2u] and\n", p->thread_id);
3827
3828         /* Structs. */
3829         ALU(t, ip, &);
3830
3831         /* Thread. */
3832         thread_ip_inc(p);
3833 }
3834
3835 static inline void
3836 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
3837 {
3838         struct thread *t = &p->threads[p->thread_id];
3839         struct instruction *ip = t->ip;
3840
3841         TRACE("[Thread %2u] and (s)\n", p->thread_id);
3842
3843         /* Structs. */
3844         ALU_S(t, ip, &);
3845
3846         /* Thread. */
3847         thread_ip_inc(p);
3848 }
3849
3850 static inline void
3851 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3852 {
3853         struct thread *t = &p->threads[p->thread_id];
3854         struct instruction *ip = t->ip;
3855
3856         TRACE("[Thread %2u] and (i)\n", p->thread_id);
3857
3858         /* Structs. */
3859         ALU_I(t, ip, &);
3860
3861         /* Thread. */
3862         thread_ip_inc(p);
3863 }
3864
3865 static inline void
3866 instr_alu_or_exec(struct rte_swx_pipeline *p)
3867 {
3868         struct thread *t = &p->threads[p->thread_id];
3869         struct instruction *ip = t->ip;
3870
3871         TRACE("[Thread %2u] or\n", p->thread_id);
3872
3873         /* Structs. */
3874         ALU(t, ip, |);
3875
3876         /* Thread. */
3877         thread_ip_inc(p);
3878 }
3879
3880 static inline void
3881 instr_alu_or_s_exec(struct rte_swx_pipeline *p)
3882 {
3883         struct thread *t = &p->threads[p->thread_id];
3884         struct instruction *ip = t->ip;
3885
3886         TRACE("[Thread %2u] or (s)\n", p->thread_id);
3887
3888         /* Structs. */
3889         ALU_S(t, ip, |);
3890
3891         /* Thread. */
3892         thread_ip_inc(p);
3893 }
3894
3895 static inline void
3896 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3897 {
3898         struct thread *t = &p->threads[p->thread_id];
3899         struct instruction *ip = t->ip;
3900
3901         TRACE("[Thread %2u] or (i)\n", p->thread_id);
3902
3903         /* Structs. */
3904         ALU_I(t, ip, |);
3905
3906         /* Thread. */
3907         thread_ip_inc(p);
3908 }
3909
3910 static inline void
3911 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3912 {
3913         struct thread *t = &p->threads[p->thread_id];
3914         struct instruction *ip = t->ip;
3915
3916         TRACE("[Thread %2u] xor\n", p->thread_id);
3917
3918         /* Structs. */
3919         ALU(t, ip, ^);
3920
3921         /* Thread. */
3922         thread_ip_inc(p);
3923 }
3924
3925 static inline void
3926 instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
3927 {
3928         struct thread *t = &p->threads[p->thread_id];
3929         struct instruction *ip = t->ip;
3930
3931         TRACE("[Thread %2u] xor (s)\n", p->thread_id);
3932
3933         /* Structs. */
3934         ALU_S(t, ip, ^);
3935
3936         /* Thread. */
3937         thread_ip_inc(p);
3938 }
3939
3940 static inline void
3941 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3942 {
3943         struct thread *t = &p->threads[p->thread_id];
3944         struct instruction *ip = t->ip;
3945
3946         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3947
3948         /* Structs. */
3949         ALU_I(t, ip, ^);
3950
3951         /* Thread. */
3952         thread_ip_inc(p);
3953 }
3954
3955 static inline void
3956 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3957 {
3958         struct thread *t = &p->threads[p->thread_id];
3959         struct instruction *ip = t->ip;
3960         uint8_t *dst_struct, *src_struct;
3961         uint16_t *dst16_ptr, dst;
3962         uint64_t *src64_ptr, src64, src64_mask, src;
3963         uint64_t r;
3964
3965         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3966
3967         /* Structs. */
3968         dst_struct = t->structs[ip->alu.dst.struct_id];
3969         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3970         dst = *dst16_ptr;
3971
3972         src_struct = t->structs[ip->alu.src.struct_id];
3973         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3974         src64 = *src64_ptr;
3975         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3976         src = src64 & src64_mask;
3977
3978         r = dst;
3979         r = ~r & 0xFFFF;
3980
3981         /* The first input (r) is a 16-bit number. The second and the third
3982          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3983          * three numbers (output r) is a 34-bit number.
3984          */
3985         r += (src >> 32) + (src & 0xFFFFFFFF);
3986
3987         /* The first input is a 16-bit number. The second input is an 18-bit
3988          * number. In the worst case scenario, the sum of the two numbers is a
3989          * 19-bit number.
3990          */
3991         r = (r & 0xFFFF) + (r >> 16);
3992
3993         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3994          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3995          */
3996         r = (r & 0xFFFF) + (r >> 16);
3997
3998         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3999          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4000          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
4001          * therefore the output r is always a 16-bit number.
4002          */
4003         r = (r & 0xFFFF) + (r >> 16);
4004
4005         r = ~r & 0xFFFF;
4006         r = r ? r : 0xFFFF;
4007
4008         *dst16_ptr = (uint16_t)r;
4009
4010         /* Thread. */
4011         thread_ip_inc(p);
4012 }
4013
4014 static inline void
4015 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4016 {
4017         struct thread *t = &p->threads[p->thread_id];
4018         struct instruction *ip = t->ip;
4019         uint8_t *dst_struct, *src_struct;
4020         uint16_t *dst16_ptr, dst;
4021         uint64_t *src64_ptr, src64, src64_mask, src;
4022         uint64_t r;
4023
4024         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
4025
4026         /* Structs. */
4027         dst_struct = t->structs[ip->alu.dst.struct_id];
4028         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4029         dst = *dst16_ptr;
4030
4031         src_struct = t->structs[ip->alu.src.struct_id];
4032         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4033         src64 = *src64_ptr;
4034         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4035         src = src64 & src64_mask;
4036
4037         r = dst;
4038         r = ~r & 0xFFFF;
4039
4040         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
4041          * the following sequence of operations in 2's complement arithmetic:
4042          *    a '- b = (a - b) % 0xFFFF.
4043          *
4044          * In order to prevent an underflow for the below subtraction, in which
4045          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
4046          * minuend), we first add a multiple of the 0xFFFF modulus to the
4047          * minuend. The number we add to the minuend needs to be a 34-bit number
4048          * or higher, so for readability reasons we picked the 36-bit multiple.
4049          * We are effectively turning the 16-bit minuend into a 36-bit number:
4050          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
4051          */
4052         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
4053
4054         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
4055          * result (the output r) is a 36-bit number.
4056          */
4057         r -= (src >> 32) + (src & 0xFFFFFFFF);
4058
4059         /* The first input is a 16-bit number. The second input is a 20-bit
4060          * number. Their sum is a 21-bit number.
4061          */
4062         r = (r & 0xFFFF) + (r >> 16);
4063
4064         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4065          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
4066          */
4067         r = (r & 0xFFFF) + (r >> 16);
4068
4069         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4070          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4071          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4072          * generated, therefore the output r is always a 16-bit number.
4073          */
4074         r = (r & 0xFFFF) + (r >> 16);
4075
4076         r = ~r & 0xFFFF;
4077         r = r ? r : 0xFFFF;
4078
4079         *dst16_ptr = (uint16_t)r;
4080
4081         /* Thread. */
4082         thread_ip_inc(p);
4083 }
4084
4085 static inline void
4086 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4087 {
4088         struct thread *t = &p->threads[p->thread_id];
4089         struct instruction *ip = t->ip;
4090         uint8_t *dst_struct, *src_struct;
4091         uint16_t *dst16_ptr;
4092         uint32_t *src32_ptr;
4093         uint64_t r0, r1;
4094
4095         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
4096
4097         /* Structs. */
4098         dst_struct = t->structs[ip->alu.dst.struct_id];
4099         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4100
4101         src_struct = t->structs[ip->alu.src.struct_id];
4102         src32_ptr = (uint32_t *)&src_struct[0];
4103
4104         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
4105         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
4106         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
4107         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
4108         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
4109
4110         /* The first input is a 16-bit number. The second input is a 19-bit
4111          * number. Their sum is a 20-bit number.
4112          */
4113         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4114
4115         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4116          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
4117          */
4118         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4119
4120         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4121          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4122          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
4123          * generated, therefore the output r is always a 16-bit number.
4124          */
4125         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4126
4127         r0 = ~r0 & 0xFFFF;
4128         r0 = r0 ? r0 : 0xFFFF;
4129
4130         *dst16_ptr = (uint16_t)r0;
4131
4132         /* Thread. */
4133         thread_ip_inc(p);
4134 }
4135
4136 static inline void
4137 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4138 {
4139         struct thread *t = &p->threads[p->thread_id];
4140         struct instruction *ip = t->ip;
4141         uint8_t *dst_struct, *src_struct;
4142         uint16_t *dst16_ptr;
4143         uint32_t *src32_ptr;
4144         uint64_t r = 0;
4145         uint32_t i;
4146
4147         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
4148
4149         /* Structs. */
4150         dst_struct = t->structs[ip->alu.dst.struct_id];
4151         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4152
4153         src_struct = t->structs[ip->alu.src.struct_id];
4154         src32_ptr = (uint32_t *)&src_struct[0];
4155
4156         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
4157          * Therefore, in the worst case scenario, a 35-bit number is added to a
4158          * 16-bit number (the input r), so the output r is 36-bit number.
4159          */
4160         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
4161                 r += *src32_ptr;
4162
4163         /* The first input is a 16-bit number. The second input is a 20-bit
4164          * number. Their sum is a 21-bit number.
4165          */
4166         r = (r & 0xFFFF) + (r >> 16);
4167
4168         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4169          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
4170          */
4171         r = (r & 0xFFFF) + (r >> 16);
4172
4173         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4174          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4175          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4176          * generated, therefore the output r is always a 16-bit number.
4177          */
4178         r = (r & 0xFFFF) + (r >> 16);
4179
4180         r = ~r & 0xFFFF;
4181         r = r ? r : 0xFFFF;
4182
4183         *dst16_ptr = (uint16_t)r;
4184
4185         /* Thread. */
4186         thread_ip_inc(p);
4187 }
4188
4189 #define RTE_SWX_INSTRUCTION_TOKENS_MAX 16
4190
4191 static int
4192 instr_translate(struct rte_swx_pipeline *p,
4193                 struct action *action,
4194                 char *string,
4195                 struct instruction *instr,
4196                 struct instruction_data *data)
4197 {
4198         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
4199         int n_tokens = 0, tpos = 0;
4200
4201         /* Parse the instruction string into tokens. */
4202         for ( ; ; ) {
4203                 char *token;
4204
4205                 token = strtok_r(string, " \t\v", &string);
4206                 if (!token)
4207                         break;
4208
4209                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
4210
4211                 tokens[n_tokens] = token;
4212                 n_tokens++;
4213         }
4214
4215         CHECK(n_tokens, EINVAL);
4216
4217         /* Handle the optional instruction label. */
4218         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
4219                 strcpy(data->label, tokens[0]);
4220
4221                 tpos += 2;
4222                 CHECK(n_tokens - tpos, EINVAL);
4223         }
4224
4225         /* Identify the instruction type. */
4226         if (!strcmp(tokens[tpos], "rx"))
4227                 return instr_rx_translate(p,
4228                                           action,
4229                                           &tokens[tpos],
4230                                           n_tokens - tpos,
4231                                           instr,
4232                                           data);
4233
4234         if (!strcmp(tokens[tpos], "tx"))
4235                 return instr_tx_translate(p,
4236                                           action,
4237                                           &tokens[tpos],
4238                                           n_tokens - tpos,
4239                                           instr,
4240                                           data);
4241
4242         if (!strcmp(tokens[tpos], "extract"))
4243                 return instr_hdr_extract_translate(p,
4244                                                    action,
4245                                                    &tokens[tpos],
4246                                                    n_tokens - tpos,
4247                                                    instr,
4248                                                    data);
4249
4250         if (!strcmp(tokens[tpos], "emit"))
4251                 return instr_hdr_emit_translate(p,
4252                                                 action,
4253                                                 &tokens[tpos],
4254                                                 n_tokens - tpos,
4255                                                 instr,
4256                                                 data);
4257
4258         if (!strcmp(tokens[tpos], "validate"))
4259                 return instr_hdr_validate_translate(p,
4260                                                     action,
4261                                                     &tokens[tpos],
4262                                                     n_tokens - tpos,
4263                                                     instr,
4264                                                     data);
4265
4266         if (!strcmp(tokens[tpos], "invalidate"))
4267                 return instr_hdr_invalidate_translate(p,
4268                                                       action,
4269                                                       &tokens[tpos],
4270                                                       n_tokens - tpos,
4271                                                       instr,
4272                                                       data);
4273
4274         if (!strcmp(tokens[tpos], "mov"))
4275                 return instr_mov_translate(p,
4276                                            action,
4277                                            &tokens[tpos],
4278                                            n_tokens - tpos,
4279                                            instr,
4280                                            data);
4281
4282         if (!strcmp(tokens[tpos], "dma"))
4283                 return instr_dma_translate(p,
4284                                            action,
4285                                            &tokens[tpos],
4286                                            n_tokens - tpos,
4287                                            instr,
4288                                            data);
4289
4290         if (!strcmp(tokens[tpos], "add"))
4291                 return instr_alu_add_translate(p,
4292                                                action,
4293                                                &tokens[tpos],
4294                                                n_tokens - tpos,
4295                                                instr,
4296                                                data);
4297
4298         if (!strcmp(tokens[tpos], "sub"))
4299                 return instr_alu_sub_translate(p,
4300                                                action,
4301                                                &tokens[tpos],
4302                                                n_tokens - tpos,
4303                                                instr,
4304                                                data);
4305
4306         if (!strcmp(tokens[tpos], "ckadd"))
4307                 return instr_alu_ckadd_translate(p,
4308                                                  action,
4309                                                  &tokens[tpos],
4310                                                  n_tokens - tpos,
4311                                                  instr,
4312                                                  data);
4313
4314         if (!strcmp(tokens[tpos], "cksub"))
4315                 return instr_alu_cksub_translate(p,
4316                                                  action,
4317                                                  &tokens[tpos],
4318                                                  n_tokens - tpos,
4319                                                  instr,
4320                                                  data);
4321
4322         if (!strcmp(tokens[tpos], "and"))
4323                 return instr_alu_and_translate(p,
4324                                                action,
4325                                                &tokens[tpos],
4326                                                n_tokens - tpos,
4327                                                instr,
4328                                                data);
4329
4330         if (!strcmp(tokens[tpos], "or"))
4331                 return instr_alu_or_translate(p,
4332                                               action,
4333                                               &tokens[tpos],
4334                                               n_tokens - tpos,
4335                                               instr,
4336                                               data);
4337
4338         if (!strcmp(tokens[tpos], "xor"))
4339                 return instr_alu_xor_translate(p,
4340                                                action,
4341                                                &tokens[tpos],
4342                                                n_tokens - tpos,
4343                                                instr,
4344                                                data);
4345
4346         if (!strcmp(tokens[tpos], "shl"))
4347                 return instr_alu_shl_translate(p,
4348                                                action,
4349                                                &tokens[tpos],
4350                                                n_tokens - tpos,
4351                                                instr,
4352                                                data);
4353
4354         if (!strcmp(tokens[tpos], "shr"))
4355                 return instr_alu_shr_translate(p,
4356                                                action,
4357                                                &tokens[tpos],
4358                                                n_tokens - tpos,
4359                                                instr,
4360                                                data);
4361
4362         if (!strcmp(tokens[tpos], "table"))
4363                 return instr_table_translate(p,
4364                                              action,
4365                                              &tokens[tpos],
4366                                              n_tokens - tpos,
4367                                              instr,
4368                                              data);
4369
4370         CHECK(0, EINVAL);
4371 }
4372
4373 static uint32_t
4374 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
4375 {
4376         uint32_t count = 0, i;
4377
4378         if (!label[0])
4379                 return 0;
4380
4381         for (i = 0; i < n; i++)
4382                 if (!strcmp(label, data[i].jmp_label))
4383                         count++;
4384
4385         return count;
4386 }
4387
4388 static int
4389 instr_label_check(struct instruction_data *instruction_data,
4390                   uint32_t n_instructions)
4391 {
4392         uint32_t i;
4393
4394         /* Check that all instruction labels are unique. */
4395         for (i = 0; i < n_instructions; i++) {
4396                 struct instruction_data *data = &instruction_data[i];
4397                 char *label = data->label;
4398                 uint32_t j;
4399
4400                 if (!label[0])
4401                         continue;
4402
4403                 for (j = i + 1; j < n_instructions; j++)
4404                         CHECK(strcmp(label, data[j].label), EINVAL);
4405         }
4406
4407         /* Get users for each instruction label. */
4408         for (i = 0; i < n_instructions; i++) {
4409                 struct instruction_data *data = &instruction_data[i];
4410                 char *label = data->label;
4411
4412                 data->n_users = label_is_used(instruction_data,
4413                                               n_instructions,
4414                                               label);
4415         }
4416
4417         return 0;
4418 }
4419
4420 static int
4421 instruction_config(struct rte_swx_pipeline *p,
4422                    struct action *a,
4423                    const char **instructions,
4424                    uint32_t n_instructions)
4425 {
4426         struct instruction *instr = NULL;
4427         struct instruction_data *data = NULL;
4428         char *string = NULL;
4429         int err = 0;
4430         uint32_t i;
4431
4432         CHECK(n_instructions, EINVAL);
4433         CHECK(instructions, EINVAL);
4434         for (i = 0; i < n_instructions; i++)
4435                 CHECK(instructions[i], EINVAL);
4436
4437         /* Memory allocation. */
4438         instr = calloc(n_instructions, sizeof(struct instruction));
4439         if (!instr) {
4440                 err = ENOMEM;
4441                 goto error;
4442         }
4443
4444         data = calloc(n_instructions, sizeof(struct instruction_data));
4445         if (!data) {
4446                 err = ENOMEM;
4447                 goto error;
4448         }
4449
4450         for (i = 0; i < n_instructions; i++) {
4451                 string = strdup(instructions[i]);
4452                 if (!string) {
4453                         err = ENOMEM;
4454                         goto error;
4455                 }
4456
4457                 err = instr_translate(p, a, string, &instr[i], &data[i]);
4458                 if (err)
4459                         goto error;
4460
4461                 free(string);
4462         }
4463
4464         err = instr_label_check(data, n_instructions);
4465         if (err)
4466                 goto error;
4467
4468         free(data);
4469
4470         if (a) {
4471                 a->instructions = instr;
4472                 a->n_instructions = n_instructions;
4473         } else {
4474                 p->instructions = instr;
4475                 p->n_instructions = n_instructions;
4476         }
4477
4478         return 0;
4479
4480 error:
4481         free(string);
4482         free(data);
4483         free(instr);
4484         return err;
4485 }
4486
4487 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
4488
4489 static instr_exec_t instruction_table[] = {
4490         [INSTR_RX] = instr_rx_exec,
4491         [INSTR_TX] = instr_tx_exec,
4492
4493         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
4494         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
4495         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
4496         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
4497         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
4498         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
4499         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
4500         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
4501
4502         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
4503         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
4504         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
4505         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
4506         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
4507         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
4508         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
4509         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
4510         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
4511
4512         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
4513         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
4514
4515         [INSTR_MOV] = instr_mov_exec,
4516         [INSTR_MOV_S] = instr_mov_s_exec,
4517         [INSTR_MOV_I] = instr_mov_i_exec,
4518
4519         [INSTR_DMA_HT] = instr_dma_ht_exec,
4520         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
4521         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
4522         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
4523         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
4524         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
4525         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
4526         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
4527
4528         [INSTR_ALU_ADD] = instr_alu_add_exec,
4529         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
4530         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
4531         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
4532         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
4533         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
4534
4535         [INSTR_ALU_SUB] = instr_alu_sub_exec,
4536         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
4537         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
4538         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
4539         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
4540         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
4541
4542         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
4543         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
4544         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
4545         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
4546
4547         [INSTR_ALU_AND] = instr_alu_and_exec,
4548         [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
4549         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
4550
4551         [INSTR_ALU_OR] = instr_alu_or_exec,
4552         [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
4553         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
4554
4555         [INSTR_ALU_XOR] = instr_alu_xor_exec,
4556         [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec,
4557         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
4558
4559         [INSTR_ALU_SHL] = instr_alu_shl_exec,
4560         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
4561         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
4562         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
4563         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
4564         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
4565
4566         [INSTR_ALU_SHR] = instr_alu_shr_exec,
4567         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
4568         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
4569         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
4570         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
4571         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
4572
4573         [INSTR_TABLE] = instr_table_exec,
4574 };
4575
4576 static inline void
4577 instr_exec(struct rte_swx_pipeline *p)
4578 {
4579         struct thread *t = &p->threads[p->thread_id];
4580         struct instruction *ip = t->ip;
4581         instr_exec_t instr = instruction_table[ip->type];
4582
4583         instr(p);
4584 }
4585
4586 /*
4587  * Action.
4588  */
4589 static struct action *
4590 action_find(struct rte_swx_pipeline *p, const char *name)
4591 {
4592         struct action *elem;
4593
4594         if (!name)
4595                 return NULL;
4596
4597         TAILQ_FOREACH(elem, &p->actions, node)
4598                 if (strcmp(elem->name, name) == 0)
4599                         return elem;
4600
4601         return NULL;
4602 }
4603
4604 static struct field *
4605 action_field_find(struct action *a, const char *name)
4606 {
4607         return a->st ? struct_type_field_find(a->st, name) : NULL;
4608 }
4609
4610 static struct field *
4611 action_field_parse(struct action *action, const char *name)
4612 {
4613         if (name[0] != 't' || name[1] != '.')
4614                 return NULL;
4615
4616         return action_field_find(action, &name[2]);
4617 }
4618
4619 int
4620 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
4621                                const char *name,
4622                                const char *args_struct_type_name,
4623                                const char **instructions,
4624                                uint32_t n_instructions)
4625 {
4626         struct struct_type *args_struct_type;
4627         struct action *a;
4628         int err;
4629
4630         CHECK(p, EINVAL);
4631
4632         CHECK_NAME(name, EINVAL);
4633         CHECK(!action_find(p, name), EEXIST);
4634
4635         if (args_struct_type_name) {
4636                 CHECK_NAME(args_struct_type_name, EINVAL);
4637                 args_struct_type = struct_type_find(p, args_struct_type_name);
4638                 CHECK(args_struct_type, EINVAL);
4639         } else {
4640                 args_struct_type = NULL;
4641         }
4642
4643         /* Node allocation. */
4644         a = calloc(1, sizeof(struct action));
4645         CHECK(a, ENOMEM);
4646
4647         /* Node initialization. */
4648         strcpy(a->name, name);
4649         a->st = args_struct_type;
4650         a->id = p->n_actions;
4651
4652         /* Instruction translation. */
4653         err = instruction_config(p, a, instructions, n_instructions);
4654         if (err) {
4655                 free(a);
4656                 return err;
4657         }
4658
4659         /* Node add to tailq. */
4660         TAILQ_INSERT_TAIL(&p->actions, a, node);
4661         p->n_actions++;
4662
4663         return 0;
4664 }
4665
4666 static int
4667 action_build(struct rte_swx_pipeline *p)
4668 {
4669         struct action *action;
4670
4671         p->action_instructions = calloc(p->n_actions,
4672                                         sizeof(struct instruction *));
4673         CHECK(p->action_instructions, ENOMEM);
4674
4675         TAILQ_FOREACH(action, &p->actions, node)
4676                 p->action_instructions[action->id] = action->instructions;
4677
4678         return 0;
4679 }
4680
4681 static void
4682 action_build_free(struct rte_swx_pipeline *p)
4683 {
4684         free(p->action_instructions);
4685         p->action_instructions = NULL;
4686 }
4687
4688 static void
4689 action_free(struct rte_swx_pipeline *p)
4690 {
4691         action_build_free(p);
4692
4693         for ( ; ; ) {
4694                 struct action *action;
4695
4696                 action = TAILQ_FIRST(&p->actions);
4697                 if (!action)
4698                         break;
4699
4700                 TAILQ_REMOVE(&p->actions, action, node);
4701                 free(action->instructions);
4702                 free(action);
4703         }
4704 }
4705
4706 /*
4707  * Table.
4708  */
4709 static struct table_type *
4710 table_type_find(struct rte_swx_pipeline *p, const char *name)
4711 {
4712         struct table_type *elem;
4713
4714         TAILQ_FOREACH(elem, &p->table_types, node)
4715                 if (strcmp(elem->name, name) == 0)
4716                         return elem;
4717
4718         return NULL;
4719 }
4720
4721 static struct table_type *
4722 table_type_resolve(struct rte_swx_pipeline *p,
4723                    const char *recommended_type_name,
4724                    enum rte_swx_table_match_type match_type)
4725 {
4726         struct table_type *elem;
4727
4728         /* Only consider the recommended type if the match type is correct. */
4729         if (recommended_type_name)
4730                 TAILQ_FOREACH(elem, &p->table_types, node)
4731                         if (!strcmp(elem->name, recommended_type_name) &&
4732                             (elem->match_type == match_type))
4733                                 return elem;
4734
4735         /* Ignore the recommended type and get the first element with this match
4736          * type.
4737          */
4738         TAILQ_FOREACH(elem, &p->table_types, node)
4739                 if (elem->match_type == match_type)
4740                         return elem;
4741
4742         return NULL;
4743 }
4744
4745 static struct table *
4746 table_find(struct rte_swx_pipeline *p, const char *name)
4747 {
4748         struct table *elem;
4749
4750         TAILQ_FOREACH(elem, &p->tables, node)
4751                 if (strcmp(elem->name, name) == 0)
4752                         return elem;
4753
4754         return NULL;
4755 }
4756
4757 static struct table *
4758 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
4759 {
4760         struct table *table = NULL;
4761
4762         TAILQ_FOREACH(table, &p->tables, node)
4763                 if (table->id == id)
4764                         return table;
4765
4766         return NULL;
4767 }
4768
4769 int
4770 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
4771                                      const char *name,
4772                                      enum rte_swx_table_match_type match_type,
4773                                      struct rte_swx_table_ops *ops)
4774 {
4775         struct table_type *elem;
4776
4777         CHECK(p, EINVAL);
4778
4779         CHECK_NAME(name, EINVAL);
4780         CHECK(!table_type_find(p, name), EEXIST);
4781
4782         CHECK(ops, EINVAL);
4783         CHECK(ops->create, EINVAL);
4784         CHECK(ops->lkp, EINVAL);
4785         CHECK(ops->free, EINVAL);
4786
4787         /* Node allocation. */
4788         elem = calloc(1, sizeof(struct table_type));
4789         CHECK(elem, ENOMEM);
4790
4791         /* Node initialization. */
4792         strcpy(elem->name, name);
4793         elem->match_type = match_type;
4794         memcpy(&elem->ops, ops, sizeof(*ops));
4795
4796         /* Node add to tailq. */
4797         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
4798
4799         return 0;
4800 }
4801
4802 static enum rte_swx_table_match_type
4803 table_match_type_resolve(struct rte_swx_match_field_params *fields,
4804                          uint32_t n_fields)
4805 {
4806         uint32_t i;
4807
4808         for (i = 0; i < n_fields; i++)
4809                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
4810                         break;
4811
4812         if (i == n_fields)
4813                 return RTE_SWX_TABLE_MATCH_EXACT;
4814
4815         if ((i == n_fields - 1) &&
4816             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
4817                 return RTE_SWX_TABLE_MATCH_LPM;
4818
4819         return RTE_SWX_TABLE_MATCH_WILDCARD;
4820 }
4821
4822 int
4823 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
4824                               const char *name,
4825                               struct rte_swx_pipeline_table_params *params,
4826                               const char *recommended_table_type_name,
4827                               const char *args,
4828                               uint32_t size)
4829 {
4830         struct table_type *type;
4831         struct table *t;
4832         struct action *default_action;
4833         struct header *header = NULL;
4834         int is_header = 0;
4835         uint32_t offset_prev = 0, action_data_size_max = 0, i;
4836
4837         CHECK(p, EINVAL);
4838
4839         CHECK_NAME(name, EINVAL);
4840         CHECK(!table_find(p, name), EEXIST);
4841
4842         CHECK(params, EINVAL);
4843
4844         /* Match checks. */
4845         CHECK(!params->n_fields || params->fields, EINVAL);
4846         for (i = 0; i < params->n_fields; i++) {
4847                 struct rte_swx_match_field_params *field = &params->fields[i];
4848                 struct header *h;
4849                 struct field *hf, *mf;
4850                 uint32_t offset;
4851
4852                 CHECK_NAME(field->name, EINVAL);
4853
4854                 hf = header_field_parse(p, field->name, &h);
4855                 mf = metadata_field_parse(p, field->name);
4856                 CHECK(hf || mf, EINVAL);
4857
4858                 offset = hf ? hf->offset : mf->offset;
4859
4860                 if (i == 0) {
4861                         is_header = hf ? 1 : 0;
4862                         header = hf ? h : NULL;
4863                         offset_prev = offset;
4864
4865                         continue;
4866                 }
4867
4868                 CHECK((is_header && hf && (h->id == header->id)) ||
4869                       (!is_header && mf), EINVAL);
4870
4871                 CHECK(offset > offset_prev, EINVAL);
4872                 offset_prev = offset;
4873         }
4874
4875         /* Action checks. */
4876         CHECK(params->n_actions, EINVAL);
4877         CHECK(params->action_names, EINVAL);
4878         for (i = 0; i < params->n_actions; i++) {
4879                 const char *action_name = params->action_names[i];
4880                 struct action *a;
4881                 uint32_t action_data_size;
4882
4883                 CHECK(action_name, EINVAL);
4884
4885                 a = action_find(p, action_name);
4886                 CHECK(a, EINVAL);
4887
4888                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
4889                 if (action_data_size > action_data_size_max)
4890                         action_data_size_max = action_data_size;
4891         }
4892
4893         CHECK(params->default_action_name, EINVAL);
4894         for (i = 0; i < p->n_actions; i++)
4895                 if (!strcmp(params->action_names[i],
4896                             params->default_action_name))
4897                         break;
4898         CHECK(i < params->n_actions, EINVAL);
4899         default_action = action_find(p, params->default_action_name);
4900         CHECK((default_action->st && params->default_action_data) ||
4901               !params->default_action_data, EINVAL);
4902
4903         /* Table type checks. */
4904         if (params->n_fields) {
4905                 enum rte_swx_table_match_type match_type;
4906
4907                 match_type = table_match_type_resolve(params->fields,
4908                                                       params->n_fields);
4909                 type = table_type_resolve(p,
4910                                           recommended_table_type_name,
4911                                           match_type);
4912                 CHECK(type, EINVAL);
4913         } else {
4914                 type = NULL;
4915         }
4916
4917         /* Memory allocation. */
4918         t = calloc(1, sizeof(struct table));
4919         CHECK(t, ENOMEM);
4920
4921         t->fields = calloc(params->n_fields, sizeof(struct match_field));
4922         if (!t->fields) {
4923                 free(t);
4924                 CHECK(0, ENOMEM);
4925         }
4926
4927         t->actions = calloc(params->n_actions, sizeof(struct action *));
4928         if (!t->actions) {
4929                 free(t->fields);
4930                 free(t);
4931                 CHECK(0, ENOMEM);
4932         }
4933
4934         if (action_data_size_max) {
4935                 t->default_action_data = calloc(1, action_data_size_max);
4936                 if (!t->default_action_data) {
4937                         free(t->actions);
4938                         free(t->fields);
4939                         free(t);
4940                         CHECK(0, ENOMEM);
4941                 }
4942         }
4943
4944         /* Node initialization. */
4945         strcpy(t->name, name);
4946         if (args && args[0])
4947                 strcpy(t->args, args);
4948         t->type = type;
4949
4950         for (i = 0; i < params->n_fields; i++) {
4951                 struct rte_swx_match_field_params *field = &params->fields[i];
4952                 struct match_field *f = &t->fields[i];
4953
4954                 f->match_type = field->match_type;
4955                 f->field = is_header ?
4956                         header_field_parse(p, field->name, NULL) :
4957                         metadata_field_parse(p, field->name);
4958         }
4959         t->n_fields = params->n_fields;
4960         t->is_header = is_header;
4961         t->header = header;
4962
4963         for (i = 0; i < params->n_actions; i++)
4964                 t->actions[i] = action_find(p, params->action_names[i]);
4965         t->default_action = default_action;
4966         if (default_action->st)
4967                 memcpy(t->default_action_data,
4968                        params->default_action_data,
4969                        default_action->st->n_bits / 8);
4970         t->n_actions = params->n_actions;
4971         t->default_action_is_const = params->default_action_is_const;
4972         t->action_data_size_max = action_data_size_max;
4973
4974         t->size = size;
4975         t->id = p->n_tables;
4976
4977         /* Node add to tailq. */
4978         TAILQ_INSERT_TAIL(&p->tables, t, node);
4979         p->n_tables++;
4980
4981         return 0;
4982 }
4983
4984 static struct rte_swx_table_params *
4985 table_params_get(struct table *table)
4986 {
4987         struct rte_swx_table_params *params;
4988         struct field *first, *last;
4989         uint8_t *key_mask;
4990         uint32_t key_size, key_offset, action_data_size, i;
4991
4992         /* Memory allocation. */
4993         params = calloc(1, sizeof(struct rte_swx_table_params));
4994         if (!params)
4995                 return NULL;
4996
4997         /* Key offset and size. */
4998         first = table->fields[0].field;
4999         last = table->fields[table->n_fields - 1].field;
5000         key_offset = first->offset / 8;
5001         key_size = (last->offset + last->n_bits - first->offset) / 8;
5002
5003         /* Memory allocation. */
5004         key_mask = calloc(1, key_size);
5005         if (!key_mask) {
5006                 free(params);
5007                 return NULL;
5008         }
5009
5010         /* Key mask. */
5011         for (i = 0; i < table->n_fields; i++) {
5012                 struct field *f = table->fields[i].field;
5013                 uint32_t start = (f->offset - first->offset) / 8;
5014                 size_t size = f->n_bits / 8;
5015
5016                 memset(&key_mask[start], 0xFF, size);
5017         }
5018
5019         /* Action data size. */
5020         action_data_size = 0;
5021         for (i = 0; i < table->n_actions; i++) {
5022                 struct action *action = table->actions[i];
5023                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
5024
5025                 if (ads > action_data_size)
5026                         action_data_size = ads;
5027         }
5028
5029         /* Fill in. */
5030         params->match_type = table->type->match_type;
5031         params->key_size = key_size;
5032         params->key_offset = key_offset;
5033         params->key_mask0 = key_mask;
5034         params->action_data_size = action_data_size;
5035         params->n_keys_max = table->size;
5036
5037         return params;
5038 }
5039
5040 static void
5041 table_params_free(struct rte_swx_table_params *params)
5042 {
5043         if (!params)
5044                 return;
5045
5046         free(params->key_mask0);
5047         free(params);
5048 }
5049
5050 static int
5051 table_state_build(struct rte_swx_pipeline *p)
5052 {
5053         struct table *table;
5054
5055         p->table_state = calloc(p->n_tables,
5056                                 sizeof(struct rte_swx_table_state));
5057         CHECK(p->table_state, ENOMEM);
5058
5059         TAILQ_FOREACH(table, &p->tables, node) {
5060                 struct rte_swx_table_state *ts = &p->table_state[table->id];
5061
5062                 if (table->type) {
5063                         struct rte_swx_table_params *params;
5064
5065                         /* ts->obj. */
5066                         params = table_params_get(table);
5067                         CHECK(params, ENOMEM);
5068
5069                         ts->obj = table->type->ops.create(params,
5070                                 NULL,
5071                                 table->args,
5072                                 p->numa_node);
5073
5074                         table_params_free(params);
5075                         CHECK(ts->obj, ENODEV);
5076                 }
5077
5078                 /* ts->default_action_data. */
5079                 if (table->action_data_size_max) {
5080                         ts->default_action_data =
5081                                 malloc(table->action_data_size_max);
5082                         CHECK(ts->default_action_data, ENOMEM);
5083
5084                         memcpy(ts->default_action_data,
5085                                table->default_action_data,
5086                                table->action_data_size_max);
5087                 }
5088
5089                 /* ts->default_action_id. */
5090                 ts->default_action_id = table->default_action->id;
5091         }
5092
5093         return 0;
5094 }
5095
5096 static void
5097 table_state_build_free(struct rte_swx_pipeline *p)
5098 {
5099         uint32_t i;
5100
5101         if (!p->table_state)
5102                 return;
5103
5104         for (i = 0; i < p->n_tables; i++) {
5105                 struct rte_swx_table_state *ts = &p->table_state[i];
5106                 struct table *table = table_find_by_id(p, i);
5107
5108                 /* ts->obj. */
5109                 if (table->type && ts->obj)
5110                         table->type->ops.free(ts->obj);
5111
5112                 /* ts->default_action_data. */
5113                 free(ts->default_action_data);
5114         }
5115
5116         free(p->table_state);
5117         p->table_state = NULL;
5118 }
5119
5120 static void
5121 table_state_free(struct rte_swx_pipeline *p)
5122 {
5123         table_state_build_free(p);
5124 }
5125
5126 static int
5127 table_stub_lkp(void *table __rte_unused,
5128                void *mailbox __rte_unused,
5129                uint8_t **key __rte_unused,
5130                uint64_t *action_id __rte_unused,
5131                uint8_t **action_data __rte_unused,
5132                int *hit)
5133 {
5134         *hit = 0;
5135         return 1; /* DONE. */
5136 }
5137
5138 static int
5139 table_build(struct rte_swx_pipeline *p)
5140 {
5141         uint32_t i;
5142
5143         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
5144                 struct thread *t = &p->threads[i];
5145                 struct table *table;
5146
5147                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
5148                 CHECK(t->tables, ENOMEM);
5149
5150                 TAILQ_FOREACH(table, &p->tables, node) {
5151                         struct table_runtime *r = &t->tables[table->id];
5152
5153                         if (table->type) {
5154                                 uint64_t size;
5155
5156                                 size = table->type->ops.mailbox_size_get();
5157
5158                                 /* r->func. */
5159                                 r->func = table->type->ops.lkp;
5160
5161                                 /* r->mailbox. */
5162                                 if (size) {
5163                                         r->mailbox = calloc(1, size);
5164                                         CHECK(r->mailbox, ENOMEM);
5165                                 }
5166
5167                                 /* r->key. */
5168                                 r->key = table->is_header ?
5169                                         &t->structs[table->header->struct_id] :
5170                                         &t->structs[p->metadata_struct_id];
5171                         } else {
5172                                 r->func = table_stub_lkp;
5173                         }
5174                 }
5175         }
5176
5177         return 0;
5178 }
5179
5180 static void
5181 table_build_free(struct rte_swx_pipeline *p)
5182 {
5183         uint32_t i;
5184
5185         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
5186                 struct thread *t = &p->threads[i];
5187                 uint32_t j;
5188
5189                 if (!t->tables)
5190                         continue;
5191
5192                 for (j = 0; j < p->n_tables; j++) {
5193                         struct table_runtime *r = &t->tables[j];
5194
5195                         free(r->mailbox);
5196                 }
5197
5198                 free(t->tables);
5199                 t->tables = NULL;
5200         }
5201 }
5202
5203 static void
5204 table_free(struct rte_swx_pipeline *p)
5205 {
5206         table_build_free(p);
5207
5208         /* Tables. */
5209         for ( ; ; ) {
5210                 struct table *elem;
5211
5212                 elem = TAILQ_FIRST(&p->tables);
5213                 if (!elem)
5214                         break;
5215
5216                 TAILQ_REMOVE(&p->tables, elem, node);
5217                 free(elem->fields);
5218                 free(elem->actions);
5219                 free(elem->default_action_data);
5220                 free(elem);
5221         }
5222
5223         /* Table types. */
5224         for ( ; ; ) {
5225                 struct table_type *elem;
5226
5227                 elem = TAILQ_FIRST(&p->table_types);
5228                 if (!elem)
5229                         break;
5230
5231                 TAILQ_REMOVE(&p->table_types, elem, node);
5232                 free(elem);
5233         }
5234 }
5235
5236 /*
5237  * Pipeline.
5238  */
5239 int
5240 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
5241 {
5242         struct rte_swx_pipeline *pipeline;
5243
5244         /* Check input parameters. */
5245         CHECK(p, EINVAL);
5246
5247         /* Memory allocation. */
5248         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
5249         CHECK(pipeline, ENOMEM);
5250
5251         /* Initialization. */
5252         TAILQ_INIT(&pipeline->struct_types);
5253         TAILQ_INIT(&pipeline->port_in_types);
5254         TAILQ_INIT(&pipeline->ports_in);
5255         TAILQ_INIT(&pipeline->port_out_types);
5256         TAILQ_INIT(&pipeline->ports_out);
5257         TAILQ_INIT(&pipeline->extern_types);
5258         TAILQ_INIT(&pipeline->extern_objs);
5259         TAILQ_INIT(&pipeline->extern_funcs);
5260         TAILQ_INIT(&pipeline->headers);
5261         TAILQ_INIT(&pipeline->actions);
5262         TAILQ_INIT(&pipeline->table_types);
5263         TAILQ_INIT(&pipeline->tables);
5264
5265         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
5266         pipeline->numa_node = numa_node;
5267
5268         *p = pipeline;
5269         return 0;
5270 }
5271
5272 void
5273 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
5274 {
5275         if (!p)
5276                 return;
5277
5278         free(p->instructions);
5279
5280         table_state_free(p);
5281         table_free(p);
5282         action_free(p);
5283         metadata_free(p);
5284         header_free(p);
5285         extern_func_free(p);
5286         extern_obj_free(p);
5287         port_out_free(p);
5288         port_in_free(p);
5289         struct_free(p);
5290
5291         free(p);
5292 }
5293
5294 int
5295 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
5296                                      const char **instructions,
5297                                      uint32_t n_instructions)
5298 {
5299         int err;
5300         uint32_t i;
5301
5302         err = instruction_config(p, NULL, instructions, n_instructions);
5303         if (err)
5304                 return err;
5305
5306         /* Thread instruction pointer reset. */
5307         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
5308                 struct thread *t = &p->threads[i];
5309
5310                 thread_ip_reset(p, t);
5311         }
5312
5313         return 0;
5314 }
5315
5316 int
5317 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
5318 {
5319         int status;
5320
5321         CHECK(p, EINVAL);
5322         CHECK(p->build_done == 0, EEXIST);
5323
5324         status = port_in_build(p);
5325         if (status)
5326                 goto error;
5327
5328         status = port_out_build(p);
5329         if (status)
5330                 goto error;
5331
5332         status = struct_build(p);
5333         if (status)
5334                 goto error;
5335
5336         status = extern_obj_build(p);
5337         if (status)
5338                 goto error;
5339
5340         status = extern_func_build(p);
5341         if (status)
5342                 goto error;
5343
5344         status = header_build(p);
5345         if (status)
5346                 goto error;
5347
5348         status = metadata_build(p);
5349         if (status)
5350                 goto error;
5351
5352         status = action_build(p);
5353         if (status)
5354                 goto error;
5355
5356         status = table_build(p);
5357         if (status)
5358                 goto error;
5359
5360         status = table_state_build(p);
5361         if (status)
5362                 goto error;
5363
5364         p->build_done = 1;
5365         return 0;
5366
5367 error:
5368         table_state_build_free(p);
5369         table_build_free(p);
5370         action_build_free(p);
5371         metadata_build_free(p);
5372         header_build_free(p);
5373         extern_func_build_free(p);
5374         extern_obj_build_free(p);
5375         port_out_build_free(p);
5376         port_in_build_free(p);
5377         struct_build_free(p);
5378
5379         return status;
5380 }
5381
5382 void
5383 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
5384 {
5385         uint32_t i;
5386
5387         for (i = 0; i < n_instructions; i++)
5388                 instr_exec(p);
5389 }
5390
5391 /*
5392  * Control.
5393  */
5394 int
5395 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
5396                                  struct rte_swx_table_state **table_state)
5397 {
5398         if (!p || !table_state || !p->build_done)
5399                 return -EINVAL;
5400
5401         *table_state = p->table_state;
5402         return 0;
5403 }
5404
5405 int
5406 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
5407                                  struct rte_swx_table_state *table_state)
5408 {
5409         if (!p || !table_state || !p->build_done)
5410                 return -EINVAL;
5411
5412         p->table_state = table_state;
5413         return 0;
5414 }