lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9 #include <arpa/inet.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_prefetch.h>
  13 #include <rte_byteorder.h>
  14
  15 #include "rte_swx_pipeline.h"
  16 #include "rte_swx_ctl.h"
  17
  18 #define CHECK(condition, err_code)                                             \
  19 do {                                                                           \
  20         if (!(condition))                                                      \
  21                 return -(err_code);                                            \
  22 } while (0)
  23
  24 #define CHECK_NAME(name, err_code)                                             \
  25         CHECK((name) &&                                                        \
  26               (name)[0] &&                                                     \
  27               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  28               err_code)
  29
  30 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  31         CHECK((instr) &&                                                       \
  32               (instr)[0] &&                                                    \
  33               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  34                RTE_SWX_INSTRUCTION_SIZE),                                      \
  35               err_code)
  36
  37 #ifndef TRACE_LEVEL
  38 #define TRACE_LEVEL 0
  39 #endif
  40
  41 #if TRACE_LEVEL
  42 #define TRACE(...) printf(__VA_ARGS__)
  43 #else
  44 #define TRACE(...)
  45 #endif
  46
  47 #define ntoh64(x) rte_be_to_cpu_64(x)
  48 #define hton64(x) rte_cpu_to_be_64(x)
  49
  50 /*
  51  * Struct.
  52  */
  53 struct field {
  54         char name[RTE_SWX_NAME_SIZE];
  55         uint32_t n_bits;
  56         uint32_t offset;
  57 };
  58
  59 struct struct_type {
  60         TAILQ_ENTRY(struct_type) node;
  61         char name[RTE_SWX_NAME_SIZE];
  62         struct field *fields;
  63         uint32_t n_fields;
  64         uint32_t n_bits;
  65 };
  66
  67 TAILQ_HEAD(struct_type_tailq, struct_type);
  68
  69 /*
  70  * Input port.
  71  */
  72 struct port_in_type {
  73         TAILQ_ENTRY(port_in_type) node;
  74         char name[RTE_SWX_NAME_SIZE];
  75         struct rte_swx_port_in_ops ops;
  76 };
  77
  78 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  79
  80 struct port_in {
  81         TAILQ_ENTRY(port_in) node;
  82         struct port_in_type *type;
  83         void *obj;
  84         uint32_t id;
  85 };
  86
  87 TAILQ_HEAD(port_in_tailq, port_in);
  88
  89 struct port_in_runtime {
  90         rte_swx_port_in_pkt_rx_t pkt_rx;
  91         void *obj;
  92 };
  93
  94 /*
  95  * Output port.
  96  */
  97 struct port_out_type {
  98         TAILQ_ENTRY(port_out_type) node;
  99         char name[RTE_SWX_NAME_SIZE];
 100         struct rte_swx_port_out_ops ops;
 101 };
 102
 103 TAILQ_HEAD(port_out_type_tailq, port_out_type);
 104
 105 struct port_out {
 106         TAILQ_ENTRY(port_out) node;
 107         struct port_out_type *type;
 108         void *obj;
 109         uint32_t id;
 110 };
 111
 112 TAILQ_HEAD(port_out_tailq, port_out);
 113
 114 struct port_out_runtime {
 115         rte_swx_port_out_pkt_tx_t pkt_tx;
 116         rte_swx_port_out_flush_t flush;
 117         void *obj;
 118 };
 119
 120 /*
 121  * Extern object.
 122  */
 123 struct extern_type_member_func {
 124         TAILQ_ENTRY(extern_type_member_func) node;
 125         char name[RTE_SWX_NAME_SIZE];
 126         rte_swx_extern_type_member_func_t func;
 127         uint32_t id;
 128 };
 129
 130 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 131
 132 struct extern_type {
 133         TAILQ_ENTRY(extern_type) node;
 134         char name[RTE_SWX_NAME_SIZE];
 135         struct struct_type *mailbox_struct_type;
 136         rte_swx_extern_type_constructor_t constructor;
 137         rte_swx_extern_type_destructor_t destructor;
 138         struct extern_type_member_func_tailq funcs;
 139         uint32_t n_funcs;
 140 };
 141
 142 TAILQ_HEAD(extern_type_tailq, extern_type);
 143
 144 struct extern_obj {
 145         TAILQ_ENTRY(extern_obj) node;
 146         char name[RTE_SWX_NAME_SIZE];
 147         struct extern_type *type;
 148         void *obj;
 149         uint32_t struct_id;
 150         uint32_t id;
 151 };
 152
 153 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 154
 155 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 156 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 157 #endif
 158
 159 struct extern_obj_runtime {
 160         void *obj;
 161         uint8_t *mailbox;
 162         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 163 };
 164
 165 /*
 166  * Extern function.
 167  */
 168 struct extern_func {
 169         TAILQ_ENTRY(extern_func) node;
 170         char name[RTE_SWX_NAME_SIZE];
 171         struct struct_type *mailbox_struct_type;
 172         rte_swx_extern_func_t func;
 173         uint32_t struct_id;
 174         uint32_t id;
 175 };
 176
 177 TAILQ_HEAD(extern_func_tailq, extern_func);
 178
 179 struct extern_func_runtime {
 180         uint8_t *mailbox;
 181         rte_swx_extern_func_t func;
 182 };
 183
 184 /*
 185  * Header.
 186  */
 187 struct header {
 188         TAILQ_ENTRY(header) node;
 189         char name[RTE_SWX_NAME_SIZE];
 190         struct struct_type *st;
 191         uint32_t struct_id;
 192         uint32_t id;
 193 };
 194
 195 TAILQ_HEAD(header_tailq, header);
 196
 197 struct header_runtime {
 198         uint8_t *ptr0;
 199 };
 200
 201 struct header_out_runtime {
 202         uint8_t *ptr0;
 203         uint8_t *ptr;
 204         uint32_t n_bytes;
 205 };
 206
 207 /*
 208  * Instruction.
 209  */
 210
 211 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 212  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 213  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 214  * when transferred to packet meta-data and in NBO when transferred to packet
 215  * headers.
 216  */
 217
 218 /* Notation conventions:
 219  *    -Header field: H = h.header.field (dst/src)
 220  *    -Meta-data field: M = m.field (dst/src)
 221  *    -Extern object mailbox field: E = e.field (dst/src)
 222  *    -Extern function mailbox field: F = f.field (dst/src)
 223  *    -Table action data field: T = t.field (src only)
 224  *    -Immediate value: I = 32-bit unsigned value (src only)
 225  */
 226
 227 enum instruction_type {
 228         /* rx m.port_in */
 229         INSTR_RX,
 230
 231         /* tx m.port_out */
 232         INSTR_TX,
 233
 234         /* extract h.header */
 235         INSTR_HDR_EXTRACT,
 236         INSTR_HDR_EXTRACT2,
 237         INSTR_HDR_EXTRACT3,
 238         INSTR_HDR_EXTRACT4,
 239         INSTR_HDR_EXTRACT5,
 240         INSTR_HDR_EXTRACT6,
 241         INSTR_HDR_EXTRACT7,
 242         INSTR_HDR_EXTRACT8,
 243
 244         /* emit h.header */
 245         INSTR_HDR_EMIT,
 246         INSTR_HDR_EMIT_TX,
 247         INSTR_HDR_EMIT2_TX,
 248         INSTR_HDR_EMIT3_TX,
 249         INSTR_HDR_EMIT4_TX,
 250         INSTR_HDR_EMIT5_TX,
 251         INSTR_HDR_EMIT6_TX,
 252         INSTR_HDR_EMIT7_TX,
 253         INSTR_HDR_EMIT8_TX,
 254
 255         /* validate h.header */
 256         INSTR_HDR_VALIDATE,
 257
 258         /* invalidate h.header */
 259         INSTR_HDR_INVALIDATE,
 260
 261         /* mov dst src
 262          * dst = src
 263          * dst = HMEF, src = HMEFTI
 264          */
 265         INSTR_MOV,   /* dst = MEF, src = MEFT */
 266         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 267         INSTR_MOV_I, /* dst = HMEF, src = I */
 268
 269         /* dma h.header t.field
 270          * memcpy(h.header, t.field, sizeof(h.header))
 271          */
 272         INSTR_DMA_HT,
 273         INSTR_DMA_HT2,
 274         INSTR_DMA_HT3,
 275         INSTR_DMA_HT4,
 276         INSTR_DMA_HT5,
 277         INSTR_DMA_HT6,
 278         INSTR_DMA_HT7,
 279         INSTR_DMA_HT8,
 280
 281         /* add dst src
 282          * dst += src
 283          * dst = HMEF, src = HMEFTI
 284          */
 285         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 286         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 287         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 288         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 289         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 290         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 291
 292         /* sub dst src
 293          * dst -= src
 294          * dst = HMEF, src = HMEFTI
 295          */
 296         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 297         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 298         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 299         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 300         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 301         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 302
 303         /* ckadd dst src
 304          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 305          * dst = H, src = {H, h.header}
 306          */
 307         INSTR_ALU_CKADD_FIELD,    /* src = H */
 308         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 309         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 310
 311         /* cksub dst src
 312          * dst = dst '- src
 313          * dst = H, src = H
 314          */
 315         INSTR_ALU_CKSUB_FIELD,
 316
 317         /* and dst src
 318          * dst &= src
 319          * dst = HMEF, src = HMEFTI
 320          */
 321         INSTR_ALU_AND,   /* dst = MEF, src = MEFT */
 322         INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 323         INSTR_ALU_AND_I, /* dst = HMEF, src = I */
 324
 325         /* or dst src
 326          * dst |= src
 327          * dst = HMEF, src = HMEFTI
 328          */
 329         INSTR_ALU_OR,   /* dst = MEF, src = MEFT */
 330         INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 331         INSTR_ALU_OR_I, /* dst = HMEF, src = I */
 332
 333         /* xor dst src
 334          * dst ^= src
 335          * dst = HMEF, src = HMEFTI
 336          */
 337         INSTR_ALU_XOR,   /* dst = MEF, src = MEFT */
 338         INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 339         INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
 340
 341         /* shl dst src
 342          * dst <<= src
 343          * dst = HMEF, src = HMEFTI
 344          */
 345         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 346         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 347         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 348         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 349         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 350         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 351
 352         /* shr dst src
 353          * dst >>= src
 354          * dst = HMEF, src = HMEFTI
 355          */
 356         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 357         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 358         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 359         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 360         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 361         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 362
 363         /* table TABLE */
 364         INSTR_TABLE,
 365
 366         /* extern e.obj.func */
 367         INSTR_EXTERN_OBJ,
 368
 369         /* extern f.func */
 370         INSTR_EXTERN_FUNC,
 371
 372         /* jmp LABEL
 373          * Unconditional jump
 374          */
 375         INSTR_JMP,
 376
 377         /* jmpv LABEL h.header
 378          * Jump if header is valid
 379          */
 380         INSTR_JMP_VALID,
 381
 382         /* jmpnv LABEL h.header
 383          * Jump if header is invalid
 384          */
 385         INSTR_JMP_INVALID,
 386
 387         /* jmph LABEL
 388          * Jump if table lookup hit
 389          */
 390         INSTR_JMP_HIT,
 391
 392         /* jmpnh LABEL
 393          * Jump if table lookup miss
 394          */
 395         INSTR_JMP_MISS,
 396
 397         /* jmpa LABEL ACTION
 398          * Jump if action run
 399          */
 400         INSTR_JMP_ACTION_HIT,
 401
 402         /* jmpna LABEL ACTION
 403          * Jump if action not run
 404          */
 405         INSTR_JMP_ACTION_MISS,
 406
 407         /* jmpeq LABEL a b
 408          * Jump is a is equal to b
 409          * a = HMEFT, b = HMEFTI
 410          */
 411         INSTR_JMP_EQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
 412         INSTR_JMP_EQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
 413         INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 414
 415         /* jmpneq LABEL a b
 416          * Jump is a is not equal to b
 417          * a = HMEFT, b = HMEFTI
 418          */
 419         INSTR_JMP_NEQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
 420         INSTR_JMP_NEQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
 421         INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 422
 423         /* jmplt LABEL a b
 424          * Jump if a is less than b
 425          * a = HMEFT, b = HMEFTI
 426          */
 427         INSTR_JMP_LT,    /* a = MEF, b = MEF */
 428         INSTR_JMP_LT_MH, /* a = MEF, b = H */
 429         INSTR_JMP_LT_HM, /* a = H, b = MEF */
 430         INSTR_JMP_LT_HH, /* a = H, b = H */
 431         INSTR_JMP_LT_MI, /* a = MEF, b = I */
 432         INSTR_JMP_LT_HI, /* a = H, b = I */
 433
 434         /* jmpgt LABEL a b
 435          * Jump if a is greater than b
 436          * a = HMEFT, b = HMEFTI
 437          */
 438         INSTR_JMP_GT,    /* a = MEF, b = MEF */
 439         INSTR_JMP_GT_MH, /* a = MEF, b = H */
 440         INSTR_JMP_GT_HM, /* a = H, b = MEF */
 441         INSTR_JMP_GT_HH, /* a = H, b = H */
 442         INSTR_JMP_GT_MI, /* a = MEF, b = I */
 443         INSTR_JMP_GT_HI, /* a = H, b = I */
 444
 445         /* return
 446          * Return from action
 447          */
 448         INSTR_RETURN,
 449 };
 450
 451 struct instr_operand {
 452         uint8_t struct_id;
 453         uint8_t n_bits;
 454         uint8_t offset;
 455         uint8_t pad;
 456 };
 457
 458 struct instr_io {
 459         struct {
 460                 uint8_t offset;
 461                 uint8_t n_bits;
 462                 uint8_t pad[2];
 463         } io;
 464
 465         struct {
 466                 uint8_t header_id[8];
 467                 uint8_t struct_id[8];
 468                 uint8_t n_bytes[8];
 469         } hdr;
 470 };
 471
 472 struct instr_hdr_validity {
 473         uint8_t header_id;
 474 };
 475
 476 struct instr_table {
 477         uint8_t table_id;
 478 };
 479
 480 struct instr_extern_obj {
 481         uint8_t ext_obj_id;
 482         uint8_t func_id;
 483 };
 484
 485 struct instr_extern_func {
 486         uint8_t ext_func_id;
 487 };
 488
 489 struct instr_dst_src {
 490         struct instr_operand dst;
 491         union {
 492                 struct instr_operand src;
 493                 uint32_t src_val;
 494         };
 495 };
 496
 497 struct instr_dma {
 498         struct {
 499                 uint8_t header_id[8];
 500                 uint8_t struct_id[8];
 501         } dst;
 502
 503         struct {
 504                 uint8_t offset[8];
 505         } src;
 506
 507         uint16_t n_bytes[8];
 508 };
 509
 510 struct instr_jmp {
 511         struct instruction *ip;
 512
 513         union {
 514                 struct instr_operand a;
 515                 uint8_t header_id;
 516                 uint8_t action_id;
 517         };
 518
 519         union {
 520                 struct instr_operand b;
 521                 uint32_t b_val;
 522         };
 523 };
 524
 525 struct instruction {
 526         enum instruction_type type;
 527         union {
 528                 struct instr_io io;
 529                 struct instr_hdr_validity valid;
 530                 struct instr_dst_src mov;
 531                 struct instr_dma dma;
 532                 struct instr_dst_src alu;
 533                 struct instr_table table;
 534                 struct instr_extern_obj ext_obj;
 535                 struct instr_extern_func ext_func;
 536                 struct instr_jmp jmp;
 537         };
 538 };
 539
 540 struct instruction_data {
 541         char label[RTE_SWX_NAME_SIZE];
 542         char jmp_label[RTE_SWX_NAME_SIZE];
 543         uint32_t n_users; /* user = jmp instruction to this instruction. */
 544         int invalid;
 545 };
 546
 547 /*
 548  * Action.
 549  */
 550 struct action {
 551         TAILQ_ENTRY(action) node;
 552         char name[RTE_SWX_NAME_SIZE];
 553         struct struct_type *st;
 554         struct instruction *instructions;
 555         uint32_t n_instructions;
 556         uint32_t id;
 557 };
 558
 559 TAILQ_HEAD(action_tailq, action);
 560
 561 /*
 562  * Table.
 563  */
 564 struct table_type {
 565         TAILQ_ENTRY(table_type) node;
 566         char name[RTE_SWX_NAME_SIZE];
 567         enum rte_swx_table_match_type match_type;
 568         struct rte_swx_table_ops ops;
 569 };
 570
 571 TAILQ_HEAD(table_type_tailq, table_type);
 572
 573 struct match_field {
 574         enum rte_swx_table_match_type match_type;
 575         struct field *field;
 576 };
 577
 578 struct table {
 579         TAILQ_ENTRY(table) node;
 580         char name[RTE_SWX_NAME_SIZE];
 581         char args[RTE_SWX_NAME_SIZE];
 582         struct table_type *type; /* NULL when n_fields == 0. */
 583
 584         /* Match. */
 585         struct match_field *fields;
 586         uint32_t n_fields;
 587         int is_header; /* Only valid when n_fields > 0. */
 588         struct header *header; /* Only valid when n_fields > 0. */
 589
 590         /* Action. */
 591         struct action **actions;
 592         struct action *default_action;
 593         uint8_t *default_action_data;
 594         uint32_t n_actions;
 595         int default_action_is_const;
 596         uint32_t action_data_size_max;
 597
 598         uint32_t size;
 599         uint32_t id;
 600 };
 601
 602 TAILQ_HEAD(table_tailq, table);
 603
 604 struct table_runtime {
 605         rte_swx_table_lookup_t func;
 606         void *mailbox;
 607         uint8_t **key;
 608 };
 609
 610 /*
 611  * Pipeline.
 612  */
 613 struct thread {
 614         /* Packet. */
 615         struct rte_swx_pkt pkt;
 616         uint8_t *ptr;
 617
 618         /* Structures. */
 619         uint8_t **structs;
 620
 621         /* Packet headers. */
 622         struct header_runtime *headers; /* Extracted or generated headers. */
 623         struct header_out_runtime *headers_out; /* Emitted headers. */
 624         uint8_t *header_storage;
 625         uint8_t *header_out_storage;
 626         uint64_t valid_headers;
 627         uint32_t n_headers_out;
 628
 629         /* Packet meta-data. */
 630         uint8_t *metadata;
 631
 632         /* Tables. */
 633         struct table_runtime *tables;
 634         struct rte_swx_table_state *table_state;
 635         uint64_t action_id;
 636         int hit; /* 0 = Miss, 1 = Hit. */
 637
 638         /* Extern objects and functions. */
 639         struct extern_obj_runtime *extern_objs;
 640         struct extern_func_runtime *extern_funcs;
 641
 642         /* Instructions. */
 643         struct instruction *ip;
 644         struct instruction *ret;
 645 };
 646
 647 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 648 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 649 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 650
 651 #define HEADER_VALID(thread, header_id) \
 652         MASK64_BIT_GET((thread)->valid_headers, header_id)
 653
 654 #define ALU(thread, ip, operator)  \
 655 {                                                                              \
 656         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 657         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 658         uint64_t dst64 = *dst64_ptr;                                           \
 659         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 660         uint64_t dst = dst64 & dst64_mask;                                     \
 661                                                                                \
 662         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 663         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 664         uint64_t src64 = *src64_ptr;                                           \
 665         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 666         uint64_t src = src64 & src64_mask;                                     \
 667                                                                                \
 668         uint64_t result = dst operator src;                                    \
 669                                                                                \
 670         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 671 }
 672
 673 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 674
 675 #define ALU_S(thread, ip, operator)  \
 676 {                                                                              \
 677         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 678         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 679         uint64_t dst64 = *dst64_ptr;                                           \
 680         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 681         uint64_t dst = dst64 & dst64_mask;                                     \
 682                                                                                \
 683         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 684         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 685         uint64_t src64 = *src64_ptr;                                           \
 686         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 687                                                                                \
 688         uint64_t result = dst operator src;                                    \
 689                                                                                \
 690         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 691 }
 692
 693 #define ALU_MH ALU_S
 694
 695 #define ALU_HM(thread, ip, operator)  \
 696 {                                                                              \
 697         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 698         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 699         uint64_t dst64 = *dst64_ptr;                                           \
 700         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 701         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 702                                                                                \
 703         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 704         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 705         uint64_t src64 = *src64_ptr;                                           \
 706         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 707         uint64_t src = src64 & src64_mask;                                     \
 708                                                                                \
 709         uint64_t result = dst operator src;                                    \
 710         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 711                                                                                \
 712         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 713 }
 714
 715 #define ALU_HH(thread, ip, operator)  \
 716 {                                                                              \
 717         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 718         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 719         uint64_t dst64 = *dst64_ptr;                                           \
 720         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 721         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 722                                                                                \
 723         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 724         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 725         uint64_t src64 = *src64_ptr;                                           \
 726         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 727                                                                                \
 728         uint64_t result = dst operator src;                                    \
 729         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 730                                                                                \
 731         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 732 }
 733
 734 #else
 735
 736 #define ALU_S ALU
 737 #define ALU_MH ALU
 738 #define ALU_HM ALU
 739 #define ALU_HH ALU
 740
 741 #endif
 742
 743 #define ALU_I(thread, ip, operator)  \
 744 {                                                                              \
 745         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 746         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 747         uint64_t dst64 = *dst64_ptr;                                           \
 748         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 749         uint64_t dst = dst64 & dst64_mask;                                     \
 750                                                                                \
 751         uint64_t src = (ip)->alu.src_val;                                      \
 752                                                                                \
 753         uint64_t result = dst operator src;                                    \
 754                                                                                \
 755         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 756 }
 757
 758 #define ALU_MI ALU_I
 759
 760 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 761
 762 #define ALU_HI(thread, ip, operator)  \
 763 {                                                                              \
 764         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 765         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 766         uint64_t dst64 = *dst64_ptr;                                           \
 767         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 768         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 769                                                                                \
 770         uint64_t src = (ip)->alu.src_val;                                      \
 771                                                                                \
 772         uint64_t result = dst operator src;                                    \
 773         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 774                                                                                \
 775         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 776 }
 777
 778 #else
 779
 780 #define ALU_HI ALU_I
 781
 782 #endif
 783
 784 #define MOV(thread, ip)  \
 785 {                                                                              \
 786         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 787         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 788         uint64_t dst64 = *dst64_ptr;                                           \
 789         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 790                                                                                \
 791         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 792         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 793         uint64_t src64 = *src64_ptr;                                           \
 794         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
 795         uint64_t src = src64 & src64_mask;                                     \
 796                                                                                \
 797         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 798 }
 799
 800 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 801
 802 #define MOV_S(thread, ip)  \
 803 {                                                                              \
 804         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 805         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 806         uint64_t dst64 = *dst64_ptr;                                           \
 807         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 808                                                                                \
 809         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 810         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 811         uint64_t src64 = *src64_ptr;                                           \
 812         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
 813                                                                                \
 814         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 815 }
 816
 817 #else
 818
 819 #define MOV_S MOV
 820
 821 #endif
 822
 823 #define MOV_I(thread, ip)  \
 824 {                                                                              \
 825         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 826         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 827         uint64_t dst64 = *dst64_ptr;                                           \
 828         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 829                                                                                \
 830         uint64_t src = (ip)->mov.src_val;                                      \
 831                                                                                \
 832         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 833 }
 834
 835 #define JMP_CMP(thread, ip, operator)  \
 836 {                                                                              \
 837         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
 838         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
 839         uint64_t a64 = *a64_ptr;                                               \
 840         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
 841         uint64_t a = a64 & a64_mask;                                           \
 842                                                                                \
 843         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
 844         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
 845         uint64_t b64 = *b64_ptr;                                               \
 846         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
 847         uint64_t b = b64 & b64_mask;                                           \
 848                                                                                \
 849         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
 850 }
 851
 852 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 853
 854 #define JMP_CMP_S(thread, ip, operator)  \
 855 {                                                                              \
 856         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
 857         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
 858         uint64_t a64 = *a64_ptr;                                               \
 859         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
 860         uint64_t a = a64 & a64_mask;                                           \
 861                                                                                \
 862         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
 863         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
 864         uint64_t b64 = *b64_ptr;                                               \
 865         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
 866                                                                                \
 867         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
 868 }
 869
 870 #define JMP_CMP_MH JMP_CMP_S
 871
 872 #define JMP_CMP_HM(thread, ip, operator)  \
 873 {                                                                              \
 874         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
 875         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
 876         uint64_t a64 = *a64_ptr;                                               \
 877         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
 878                                                                                \
 879         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
 880         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
 881         uint64_t b64 = *b64_ptr;                                               \
 882         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
 883         uint64_t b = b64 & b64_mask;                                           \
 884                                                                                \
 885         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
 886 }
 887
 888 #define JMP_CMP_HH(thread, ip, operator)  \
 889 {                                                                              \
 890         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
 891         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
 892         uint64_t a64 = *a64_ptr;                                               \
 893         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
 894                                                                                \
 895         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
 896         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
 897         uint64_t b64 = *b64_ptr;                                               \
 898         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
 899                                                                                \
 900         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
 901 }
 902
 903 #else
 904
 905 #define JMP_CMP_S JMP_CMP
 906 #define JMP_CMP_MH JMP_CMP
 907 #define JMP_CMP_HM JMP_CMP
 908 #define JMP_CMP_HH JMP_CMP
 909
 910 #endif
 911
 912 #define JMP_CMP_I(thread, ip, operator)  \
 913 {                                                                              \
 914         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
 915         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
 916         uint64_t a64 = *a64_ptr;                                               \
 917         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
 918         uint64_t a = a64 & a64_mask;                                           \
 919                                                                                \
 920         uint64_t b = (ip)->jmp.b_val;                                          \
 921                                                                                \
 922         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
 923 }
 924
 925 #define JMP_CMP_MI JMP_CMP_I
 926
 927 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 928
 929 #define JMP_CMP_HI(thread, ip, operator)  \
 930 {                                                                              \
 931         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
 932         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
 933         uint64_t a64 = *a64_ptr;                                               \
 934         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
 935                                                                                \
 936         uint64_t b = (ip)->jmp.b_val;                                          \
 937                                                                                \
 938         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
 939 }
 940
 941 #else
 942
 943 #define JMP_CMP_HI JMP_CMP_I
 944
 945 #endif
 946
 947 #define METADATA_READ(thread, offset, n_bits)                                  \
 948 ({                                                                             \
 949         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 950         uint64_t m64 = *m64_ptr;                                               \
 951         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 952         (m64 & m64_mask);                                                      \
 953 })
 954
 955 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
 956 {                                                                              \
 957         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 958         uint64_t m64 = *m64_ptr;                                               \
 959         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 960                                                                                \
 961         uint64_t m_new = value;                                                \
 962                                                                                \
 963         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
 964 }
 965
 966 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 967 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 968 #endif
 969
 970 struct rte_swx_pipeline {
 971         struct struct_type_tailq struct_types;
 972         struct port_in_type_tailq port_in_types;
 973         struct port_in_tailq ports_in;
 974         struct port_out_type_tailq port_out_types;
 975         struct port_out_tailq ports_out;
 976         struct extern_type_tailq extern_types;
 977         struct extern_obj_tailq extern_objs;
 978         struct extern_func_tailq extern_funcs;
 979         struct header_tailq headers;
 980         struct struct_type *metadata_st;
 981         uint32_t metadata_struct_id;
 982         struct action_tailq actions;
 983         struct table_type_tailq table_types;
 984         struct table_tailq tables;
 985
 986         struct port_in_runtime *in;
 987         struct port_out_runtime *out;
 988         struct instruction **action_instructions;
 989         struct rte_swx_table_state *table_state;
 990         struct instruction *instructions;
 991         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 992
 993         uint32_t n_structs;
 994         uint32_t n_ports_in;
 995         uint32_t n_ports_out;
 996         uint32_t n_extern_objs;
 997         uint32_t n_extern_funcs;
 998         uint32_t n_actions;
 999         uint32_t n_tables;
1000         uint32_t n_headers;
1001         uint32_t thread_id;
1002         uint32_t port_id;
1003         uint32_t n_instructions;
1004         int build_done;
1005         int numa_node;
1006 };
1007
1008 /*
1009  * Struct.
1010  */
1011 static struct struct_type *
1012 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1013 {
1014         struct struct_type *elem;
1015
1016         TAILQ_FOREACH(elem, &p->struct_types, node)
1017                 if (strcmp(elem->name, name) == 0)
1018                         return elem;
1019
1020         return NULL;
1021 }
1022
1023 static struct field *
1024 struct_type_field_find(struct struct_type *st, const char *name)
1025 {
1026         uint32_t i;
1027
1028         for (i = 0; i < st->n_fields; i++) {
1029                 struct field *f = &st->fields[i];
1030
1031                 if (strcmp(f->name, name) == 0)
1032                         return f;
1033         }
1034
1035         return NULL;
1036 }
1037
1038 int
1039 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1040                                       const char *name,
1041                                       struct rte_swx_field_params *fields,
1042                                       uint32_t n_fields)
1043 {
1044         struct struct_type *st;
1045         uint32_t i;
1046
1047         CHECK(p, EINVAL);
1048         CHECK_NAME(name, EINVAL);
1049         CHECK(fields, EINVAL);
1050         CHECK(n_fields, EINVAL);
1051
1052         for (i = 0; i < n_fields; i++) {
1053                 struct rte_swx_field_params *f = &fields[i];
1054                 uint32_t j;
1055
1056                 CHECK_NAME(f->name, EINVAL);
1057                 CHECK(f->n_bits, EINVAL);
1058                 CHECK(f->n_bits <= 64, EINVAL);
1059                 CHECK((f->n_bits & 7) == 0, EINVAL);
1060
1061                 for (j = 0; j < i; j++) {
1062                         struct rte_swx_field_params *f_prev = &fields[j];
1063
1064                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
1065                 }
1066         }
1067
1068         CHECK(!struct_type_find(p, name), EEXIST);
1069
1070         /* Node allocation. */
1071         st = calloc(1, sizeof(struct struct_type));
1072         CHECK(st, ENOMEM);
1073
1074         st->fields = calloc(n_fields, sizeof(struct field));
1075         if (!st->fields) {
1076                 free(st);
1077                 CHECK(0, ENOMEM);
1078         }
1079
1080         /* Node initialization. */
1081         strcpy(st->name, name);
1082         for (i = 0; i < n_fields; i++) {
1083                 struct field *dst = &st->fields[i];
1084                 struct rte_swx_field_params *src = &fields[i];
1085
1086                 strcpy(dst->name, src->name);
1087                 dst->n_bits = src->n_bits;
1088                 dst->offset = st->n_bits;
1089
1090                 st->n_bits += src->n_bits;
1091         }
1092         st->n_fields = n_fields;
1093
1094         /* Node add to tailq. */
1095         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1096
1097         return 0;
1098 }
1099
1100 static int
1101 struct_build(struct rte_swx_pipeline *p)
1102 {
1103         uint32_t i;
1104
1105         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1106                 struct thread *t = &p->threads[i];
1107
1108                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1109                 CHECK(t->structs, ENOMEM);
1110         }
1111
1112         return 0;
1113 }
1114
1115 static void
1116 struct_build_free(struct rte_swx_pipeline *p)
1117 {
1118         uint32_t i;
1119
1120         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1121                 struct thread *t = &p->threads[i];
1122
1123                 free(t->structs);
1124                 t->structs = NULL;
1125         }
1126 }
1127
1128 static void
1129 struct_free(struct rte_swx_pipeline *p)
1130 {
1131         struct_build_free(p);
1132
1133         /* Struct types. */
1134         for ( ; ; ) {
1135                 struct struct_type *elem;
1136
1137                 elem = TAILQ_FIRST(&p->struct_types);
1138                 if (!elem)
1139                         break;
1140
1141                 TAILQ_REMOVE(&p->struct_types, elem, node);
1142                 free(elem->fields);
1143                 free(elem);
1144         }
1145 }
1146
1147 /*
1148  * Input port.
1149  */
1150 static struct port_in_type *
1151 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1152 {
1153         struct port_in_type *elem;
1154
1155         if (!name)
1156                 return NULL;
1157
1158         TAILQ_FOREACH(elem, &p->port_in_types, node)
1159                 if (strcmp(elem->name, name) == 0)
1160                         return elem;
1161
1162         return NULL;
1163 }
1164
1165 int
1166 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1167                                        const char *name,
1168                                        struct rte_swx_port_in_ops *ops)
1169 {
1170         struct port_in_type *elem;
1171
1172         CHECK(p, EINVAL);
1173         CHECK_NAME(name, EINVAL);
1174         CHECK(ops, EINVAL);
1175         CHECK(ops->create, EINVAL);
1176         CHECK(ops->free, EINVAL);
1177         CHECK(ops->pkt_rx, EINVAL);
1178         CHECK(ops->stats_read, EINVAL);
1179
1180         CHECK(!port_in_type_find(p, name), EEXIST);
1181
1182         /* Node allocation. */
1183         elem = calloc(1, sizeof(struct port_in_type));
1184         CHECK(elem, ENOMEM);
1185
1186         /* Node initialization. */
1187         strcpy(elem->name, name);
1188         memcpy(&elem->ops, ops, sizeof(*ops));
1189
1190         /* Node add to tailq. */
1191         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1192
1193         return 0;
1194 }
1195
1196 static struct port_in *
1197 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1198 {
1199         struct port_in *port;
1200
1201         TAILQ_FOREACH(port, &p->ports_in, node)
1202                 if (port->id == port_id)
1203                         return port;
1204
1205         return NULL;
1206 }
1207
1208 int
1209 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1210                                 uint32_t port_id,
1211                                 const char *port_type_name,
1212                                 void *args)
1213 {
1214         struct port_in_type *type = NULL;
1215         struct port_in *port = NULL;
1216         void *obj = NULL;
1217
1218         CHECK(p, EINVAL);
1219
1220         CHECK(!port_in_find(p, port_id), EINVAL);
1221
1222         CHECK_NAME(port_type_name, EINVAL);
1223         type = port_in_type_find(p, port_type_name);
1224         CHECK(type, EINVAL);
1225
1226         obj = type->ops.create(args);
1227         CHECK(obj, ENODEV);
1228
1229         /* Node allocation. */
1230         port = calloc(1, sizeof(struct port_in));
1231         CHECK(port, ENOMEM);
1232
1233         /* Node initialization. */
1234         port->type = type;
1235         port->obj = obj;
1236         port->id = port_id;
1237
1238         /* Node add to tailq. */
1239         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1240         if (p->n_ports_in < port_id + 1)
1241                 p->n_ports_in = port_id + 1;
1242
1243         return 0;
1244 }
1245
1246 static int
1247 port_in_build(struct rte_swx_pipeline *p)
1248 {
1249         struct port_in *port;
1250         uint32_t i;
1251
1252         CHECK(p->n_ports_in, EINVAL);
1253         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1254
1255         for (i = 0; i < p->n_ports_in; i++)
1256                 CHECK(port_in_find(p, i), EINVAL);
1257
1258         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1259         CHECK(p->in, ENOMEM);
1260
1261         TAILQ_FOREACH(port, &p->ports_in, node) {
1262                 struct port_in_runtime *in = &p->in[port->id];
1263
1264                 in->pkt_rx = port->type->ops.pkt_rx;
1265                 in->obj = port->obj;
1266         }
1267
1268         return 0;
1269 }
1270
1271 static void
1272 port_in_build_free(struct rte_swx_pipeline *p)
1273 {
1274         free(p->in);
1275         p->in = NULL;
1276 }
1277
1278 static void
1279 port_in_free(struct rte_swx_pipeline *p)
1280 {
1281         port_in_build_free(p);
1282
1283         /* Input ports. */
1284         for ( ; ; ) {
1285                 struct port_in *port;
1286
1287                 port = TAILQ_FIRST(&p->ports_in);
1288                 if (!port)
1289                         break;
1290
1291                 TAILQ_REMOVE(&p->ports_in, port, node);
1292                 port->type->ops.free(port->obj);
1293                 free(port);
1294         }
1295
1296         /* Input port types. */
1297         for ( ; ; ) {
1298                 struct port_in_type *elem;
1299
1300                 elem = TAILQ_FIRST(&p->port_in_types);
1301                 if (!elem)
1302                         break;
1303
1304                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1305                 free(elem);
1306         }
1307 }
1308
1309 /*
1310  * Output port.
1311  */
1312 static struct port_out_type *
1313 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1314 {
1315         struct port_out_type *elem;
1316
1317         if (!name)
1318                 return NULL;
1319
1320         TAILQ_FOREACH(elem, &p->port_out_types, node)
1321                 if (!strcmp(elem->name, name))
1322                         return elem;
1323
1324         return NULL;
1325 }
1326
1327 int
1328 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1329                                         const char *name,
1330                                         struct rte_swx_port_out_ops *ops)
1331 {
1332         struct port_out_type *elem;
1333
1334         CHECK(p, EINVAL);
1335         CHECK_NAME(name, EINVAL);
1336         CHECK(ops, EINVAL);
1337         CHECK(ops->create, EINVAL);
1338         CHECK(ops->free, EINVAL);
1339         CHECK(ops->pkt_tx, EINVAL);
1340         CHECK(ops->stats_read, EINVAL);
1341
1342         CHECK(!port_out_type_find(p, name), EEXIST);
1343
1344         /* Node allocation. */
1345         elem = calloc(1, sizeof(struct port_out_type));
1346         CHECK(elem, ENOMEM);
1347
1348         /* Node initialization. */
1349         strcpy(elem->name, name);
1350         memcpy(&elem->ops, ops, sizeof(*ops));
1351
1352         /* Node add to tailq. */
1353         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1354
1355         return 0;
1356 }
1357
1358 static struct port_out *
1359 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1360 {
1361         struct port_out *port;
1362
1363         TAILQ_FOREACH(port, &p->ports_out, node)
1364                 if (port->id == port_id)
1365                         return port;
1366
1367         return NULL;
1368 }
1369
1370 int
1371 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1372                                  uint32_t port_id,
1373                                  const char *port_type_name,
1374                                  void *args)
1375 {
1376         struct port_out_type *type = NULL;
1377         struct port_out *port = NULL;
1378         void *obj = NULL;
1379
1380         CHECK(p, EINVAL);
1381
1382         CHECK(!port_out_find(p, port_id), EINVAL);
1383
1384         CHECK_NAME(port_type_name, EINVAL);
1385         type = port_out_type_find(p, port_type_name);
1386         CHECK(type, EINVAL);
1387
1388         obj = type->ops.create(args);
1389         CHECK(obj, ENODEV);
1390
1391         /* Node allocation. */
1392         port = calloc(1, sizeof(struct port_out));
1393         CHECK(port, ENOMEM);
1394
1395         /* Node initialization. */
1396         port->type = type;
1397         port->obj = obj;
1398         port->id = port_id;
1399
1400         /* Node add to tailq. */
1401         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1402         if (p->n_ports_out < port_id + 1)
1403                 p->n_ports_out = port_id + 1;
1404
1405         return 0;
1406 }
1407
1408 static int
1409 port_out_build(struct rte_swx_pipeline *p)
1410 {
1411         struct port_out *port;
1412         uint32_t i;
1413
1414         CHECK(p->n_ports_out, EINVAL);
1415
1416         for (i = 0; i < p->n_ports_out; i++)
1417                 CHECK(port_out_find(p, i), EINVAL);
1418
1419         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1420         CHECK(p->out, ENOMEM);
1421
1422         TAILQ_FOREACH(port, &p->ports_out, node) {
1423                 struct port_out_runtime *out = &p->out[port->id];
1424
1425                 out->pkt_tx = port->type->ops.pkt_tx;
1426                 out->flush = port->type->ops.flush;
1427                 out->obj = port->obj;
1428         }
1429
1430         return 0;
1431 }
1432
1433 static void
1434 port_out_build_free(struct rte_swx_pipeline *p)
1435 {
1436         free(p->out);
1437         p->out = NULL;
1438 }
1439
1440 static void
1441 port_out_free(struct rte_swx_pipeline *p)
1442 {
1443         port_out_build_free(p);
1444
1445         /* Output ports. */
1446         for ( ; ; ) {
1447                 struct port_out *port;
1448
1449                 port = TAILQ_FIRST(&p->ports_out);
1450                 if (!port)
1451                         break;
1452
1453                 TAILQ_REMOVE(&p->ports_out, port, node);
1454                 port->type->ops.free(port->obj);
1455                 free(port);
1456         }
1457
1458         /* Output port types. */
1459         for ( ; ; ) {
1460                 struct port_out_type *elem;
1461
1462                 elem = TAILQ_FIRST(&p->port_out_types);
1463                 if (!elem)
1464                         break;
1465
1466                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1467                 free(elem);
1468         }
1469 }
1470
1471 /*
1472  * Extern object.
1473  */
1474 static struct extern_type *
1475 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1476 {
1477         struct extern_type *elem;
1478
1479         TAILQ_FOREACH(elem, &p->extern_types, node)
1480                 if (strcmp(elem->name, name) == 0)
1481                         return elem;
1482
1483         return NULL;
1484 }
1485
1486 static struct extern_type_member_func *
1487 extern_type_member_func_find(struct extern_type *type, const char *name)
1488 {
1489         struct extern_type_member_func *elem;
1490
1491         TAILQ_FOREACH(elem, &type->funcs, node)
1492                 if (strcmp(elem->name, name) == 0)
1493                         return elem;
1494
1495         return NULL;
1496 }
1497
1498 static struct extern_obj *
1499 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1500 {
1501         struct extern_obj *elem;
1502
1503         TAILQ_FOREACH(elem, &p->extern_objs, node)
1504                 if (strcmp(elem->name, name) == 0)
1505                         return elem;
1506
1507         return NULL;
1508 }
1509
1510 static struct extern_type_member_func *
1511 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1512                              const char *name,
1513                              struct extern_obj **obj)
1514 {
1515         struct extern_obj *object;
1516         struct extern_type_member_func *func;
1517         char *object_name, *func_name;
1518
1519         if (name[0] != 'e' || name[1] != '.')
1520                 return NULL;
1521
1522         object_name = strdup(&name[2]);
1523         if (!object_name)
1524                 return NULL;
1525
1526         func_name = strchr(object_name, '.');
1527         if (!func_name) {
1528                 free(object_name);
1529                 return NULL;
1530         }
1531
1532         *func_name = 0;
1533         func_name++;
1534
1535         object = extern_obj_find(p, object_name);
1536         if (!object) {
1537                 free(object_name);
1538                 return NULL;
1539         }
1540
1541         func = extern_type_member_func_find(object->type, func_name);
1542         if (!func) {
1543                 free(object_name);
1544                 return NULL;
1545         }
1546
1547         if (obj)
1548                 *obj = object;
1549
1550         free(object_name);
1551         return func;
1552 }
1553
1554 static struct field *
1555 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1556                                const char *name,
1557                                struct extern_obj **object)
1558 {
1559         struct extern_obj *obj;
1560         struct field *f;
1561         char *obj_name, *field_name;
1562
1563         if ((name[0] != 'e') || (name[1] != '.'))
1564                 return NULL;
1565
1566         obj_name = strdup(&name[2]);
1567         if (!obj_name)
1568                 return NULL;
1569
1570         field_name = strchr(obj_name, '.');
1571         if (!field_name) {
1572                 free(obj_name);
1573                 return NULL;
1574         }
1575
1576         *field_name = 0;
1577         field_name++;
1578
1579         obj = extern_obj_find(p, obj_name);
1580         if (!obj) {
1581                 free(obj_name);
1582                 return NULL;
1583         }
1584
1585         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1586         if (!f) {
1587                 free(obj_name);
1588                 return NULL;
1589         }
1590
1591         if (object)
1592                 *object = obj;
1593
1594         free(obj_name);
1595         return f;
1596 }
1597
1598 int
1599 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1600         const char *name,
1601         const char *mailbox_struct_type_name,
1602         rte_swx_extern_type_constructor_t constructor,
1603         rte_swx_extern_type_destructor_t destructor)
1604 {
1605         struct extern_type *elem;
1606         struct struct_type *mailbox_struct_type;
1607
1608         CHECK(p, EINVAL);
1609
1610         CHECK_NAME(name, EINVAL);
1611         CHECK(!extern_type_find(p, name), EEXIST);
1612
1613         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1614         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1615         CHECK(mailbox_struct_type, EINVAL);
1616
1617         CHECK(constructor, EINVAL);
1618         CHECK(destructor, EINVAL);
1619
1620         /* Node allocation. */
1621         elem = calloc(1, sizeof(struct extern_type));
1622         CHECK(elem, ENOMEM);
1623
1624         /* Node initialization. */
1625         strcpy(elem->name, name);
1626         elem->mailbox_struct_type = mailbox_struct_type;
1627         elem->constructor = constructor;
1628         elem->destructor = destructor;
1629         TAILQ_INIT(&elem->funcs);
1630
1631         /* Node add to tailq. */
1632         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1633
1634         return 0;
1635 }
1636
1637 int
1638 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1639         const char *extern_type_name,
1640         const char *name,
1641         rte_swx_extern_type_member_func_t member_func)
1642 {
1643         struct extern_type *type;
1644         struct extern_type_member_func *type_member;
1645
1646         CHECK(p, EINVAL);
1647
1648         CHECK_NAME(extern_type_name, EINVAL);
1649         type = extern_type_find(p, extern_type_name);
1650         CHECK(type, EINVAL);
1651         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1652
1653         CHECK_NAME(name, EINVAL);
1654         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1655
1656         CHECK(member_func, EINVAL);
1657
1658         /* Node allocation. */
1659         type_member = calloc(1, sizeof(struct extern_type_member_func));
1660         CHECK(type_member, ENOMEM);
1661
1662         /* Node initialization. */
1663         strcpy(type_member->name, name);
1664         type_member->func = member_func;
1665         type_member->id = type->n_funcs;
1666
1667         /* Node add to tailq. */
1668         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1669         type->n_funcs++;
1670
1671         return 0;
1672 }
1673
1674 int
1675 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1676                                       const char *extern_type_name,
1677                                       const char *name,
1678                                       const char *args)
1679 {
1680         struct extern_type *type;
1681         struct extern_obj *obj;
1682         void *obj_handle;
1683
1684         CHECK(p, EINVAL);
1685
1686         CHECK_NAME(extern_type_name, EINVAL);
1687         type = extern_type_find(p, extern_type_name);
1688         CHECK(type, EINVAL);
1689
1690         CHECK_NAME(name, EINVAL);
1691         CHECK(!extern_obj_find(p, name), EEXIST);
1692
1693         /* Node allocation. */
1694         obj = calloc(1, sizeof(struct extern_obj));
1695         CHECK(obj, ENOMEM);
1696
1697         /* Object construction. */
1698         obj_handle = type->constructor(args);
1699         if (!obj_handle) {
1700                 free(obj);
1701                 CHECK(0, ENODEV);
1702         }
1703
1704         /* Node initialization. */
1705         strcpy(obj->name, name);
1706         obj->type = type;
1707         obj->obj = obj_handle;
1708         obj->struct_id = p->n_structs;
1709         obj->id = p->n_extern_objs;
1710
1711         /* Node add to tailq. */
1712         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1713         p->n_extern_objs++;
1714         p->n_structs++;
1715
1716         return 0;
1717 }
1718
1719 static int
1720 extern_obj_build(struct rte_swx_pipeline *p)
1721 {
1722         uint32_t i;
1723
1724         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1725                 struct thread *t = &p->threads[i];
1726                 struct extern_obj *obj;
1727
1728                 t->extern_objs = calloc(p->n_extern_objs,
1729                                         sizeof(struct extern_obj_runtime));
1730                 CHECK(t->extern_objs, ENOMEM);
1731
1732                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1733                         struct extern_obj_runtime *r =
1734                                 &t->extern_objs[obj->id];
1735                         struct extern_type_member_func *func;
1736                         uint32_t mailbox_size =
1737                                 obj->type->mailbox_struct_type->n_bits / 8;
1738
1739                         r->obj = obj->obj;
1740
1741                         r->mailbox = calloc(1, mailbox_size);
1742                         CHECK(r->mailbox, ENOMEM);
1743
1744                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1745                                 r->funcs[func->id] = func->func;
1746
1747                         t->structs[obj->struct_id] = r->mailbox;
1748                 }
1749         }
1750
1751         return 0;
1752 }
1753
1754 static void
1755 extern_obj_build_free(struct rte_swx_pipeline *p)
1756 {
1757         uint32_t i;
1758
1759         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1760                 struct thread *t = &p->threads[i];
1761                 uint32_t j;
1762
1763                 if (!t->extern_objs)
1764                         continue;
1765
1766                 for (j = 0; j < p->n_extern_objs; j++) {
1767                         struct extern_obj_runtime *r = &t->extern_objs[j];
1768
1769                         free(r->mailbox);
1770                 }
1771
1772                 free(t->extern_objs);
1773                 t->extern_objs = NULL;
1774         }
1775 }
1776
1777 static void
1778 extern_obj_free(struct rte_swx_pipeline *p)
1779 {
1780         extern_obj_build_free(p);
1781
1782         /* Extern objects. */
1783         for ( ; ; ) {
1784                 struct extern_obj *elem;
1785
1786                 elem = TAILQ_FIRST(&p->extern_objs);
1787                 if (!elem)
1788                         break;
1789
1790                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1791                 if (elem->obj)
1792                         elem->type->destructor(elem->obj);
1793                 free(elem);
1794         }
1795
1796         /* Extern types. */
1797         for ( ; ; ) {
1798                 struct extern_type *elem;
1799
1800                 elem = TAILQ_FIRST(&p->extern_types);
1801                 if (!elem)
1802                         break;
1803
1804                 TAILQ_REMOVE(&p->extern_types, elem, node);
1805
1806                 for ( ; ; ) {
1807                         struct extern_type_member_func *func;
1808
1809                         func = TAILQ_FIRST(&elem->funcs);
1810                         if (!func)
1811                                 break;
1812
1813                         TAILQ_REMOVE(&elem->funcs, func, node);
1814                         free(func);
1815                 }
1816
1817                 free(elem);
1818         }
1819 }
1820
1821 /*
1822  * Extern function.
1823  */
1824 static struct extern_func *
1825 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1826 {
1827         struct extern_func *elem;
1828
1829         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1830                 if (strcmp(elem->name, name) == 0)
1831                         return elem;
1832
1833         return NULL;
1834 }
1835
1836 static struct extern_func *
1837 extern_func_parse(struct rte_swx_pipeline *p,
1838                   const char *name)
1839 {
1840         if (name[0] != 'f' || name[1] != '.')
1841                 return NULL;
1842
1843         return extern_func_find(p, &name[2]);
1844 }
1845
1846 static struct field *
1847 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1848                                 const char *name,
1849                                 struct extern_func **function)
1850 {
1851         struct extern_func *func;
1852         struct field *f;
1853         char *func_name, *field_name;
1854
1855         if ((name[0] != 'f') || (name[1] != '.'))
1856                 return NULL;
1857
1858         func_name = strdup(&name[2]);
1859         if (!func_name)
1860                 return NULL;
1861
1862         field_name = strchr(func_name, '.');
1863         if (!field_name) {
1864                 free(func_name);
1865                 return NULL;
1866         }
1867
1868         *field_name = 0;
1869         field_name++;
1870
1871         func = extern_func_find(p, func_name);
1872         if (!func) {
1873                 free(func_name);
1874                 return NULL;
1875         }
1876
1877         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1878         if (!f) {
1879                 free(func_name);
1880                 return NULL;
1881         }
1882
1883         if (function)
1884                 *function = func;
1885
1886         free(func_name);
1887         return f;
1888 }
1889
1890 int
1891 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1892                                       const char *name,
1893                                       const char *mailbox_struct_type_name,
1894                                       rte_swx_extern_func_t func)
1895 {
1896         struct extern_func *f;
1897         struct struct_type *mailbox_struct_type;
1898
1899         CHECK(p, EINVAL);
1900
1901         CHECK_NAME(name, EINVAL);
1902         CHECK(!extern_func_find(p, name), EEXIST);
1903
1904         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1905         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1906         CHECK(mailbox_struct_type, EINVAL);
1907
1908         CHECK(func, EINVAL);
1909
1910         /* Node allocation. */
1911         f = calloc(1, sizeof(struct extern_func));
1912         CHECK(func, ENOMEM);
1913
1914         /* Node initialization. */
1915         strcpy(f->name, name);
1916         f->mailbox_struct_type = mailbox_struct_type;
1917         f->func = func;
1918         f->struct_id = p->n_structs;
1919         f->id = p->n_extern_funcs;
1920
1921         /* Node add to tailq. */
1922         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1923         p->n_extern_funcs++;
1924         p->n_structs++;
1925
1926         return 0;
1927 }
1928
1929 static int
1930 extern_func_build(struct rte_swx_pipeline *p)
1931 {
1932         uint32_t i;
1933
1934         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1935                 struct thread *t = &p->threads[i];
1936                 struct extern_func *func;
1937
1938                 /* Memory allocation. */
1939                 t->extern_funcs = calloc(p->n_extern_funcs,
1940                                          sizeof(struct extern_func_runtime));
1941                 CHECK(t->extern_funcs, ENOMEM);
1942
1943                 /* Extern function. */
1944                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1945                         struct extern_func_runtime *r =
1946                                 &t->extern_funcs[func->id];
1947                         uint32_t mailbox_size =
1948                                 func->mailbox_struct_type->n_bits / 8;
1949
1950                         r->func = func->func;
1951
1952                         r->mailbox = calloc(1, mailbox_size);
1953                         CHECK(r->mailbox, ENOMEM);
1954
1955                         t->structs[func->struct_id] = r->mailbox;
1956                 }
1957         }
1958
1959         return 0;
1960 }
1961
1962 static void
1963 extern_func_build_free(struct rte_swx_pipeline *p)
1964 {
1965         uint32_t i;
1966
1967         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1968                 struct thread *t = &p->threads[i];
1969                 uint32_t j;
1970
1971                 if (!t->extern_funcs)
1972                         continue;
1973
1974                 for (j = 0; j < p->n_extern_funcs; j++) {
1975                         struct extern_func_runtime *r = &t->extern_funcs[j];
1976
1977                         free(r->mailbox);
1978                 }
1979
1980                 free(t->extern_funcs);
1981                 t->extern_funcs = NULL;
1982         }
1983 }
1984
1985 static void
1986 extern_func_free(struct rte_swx_pipeline *p)
1987 {
1988         extern_func_build_free(p);
1989
1990         for ( ; ; ) {
1991                 struct extern_func *elem;
1992
1993                 elem = TAILQ_FIRST(&p->extern_funcs);
1994                 if (!elem)
1995                         break;
1996
1997                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1998                 free(elem);
1999         }
2000 }
2001
2002 /*
2003  * Header.
2004  */
2005 static struct header *
2006 header_find(struct rte_swx_pipeline *p, const char *name)
2007 {
2008         struct header *elem;
2009
2010         TAILQ_FOREACH(elem, &p->headers, node)
2011                 if (strcmp(elem->name, name) == 0)
2012                         return elem;
2013
2014         return NULL;
2015 }
2016
2017 static struct header *
2018 header_parse(struct rte_swx_pipeline *p,
2019              const char *name)
2020 {
2021         if (name[0] != 'h' || name[1] != '.')
2022                 return NULL;
2023
2024         return header_find(p, &name[2]);
2025 }
2026
2027 static struct field *
2028 header_field_parse(struct rte_swx_pipeline *p,
2029                    const char *name,
2030                    struct header **header)
2031 {
2032         struct header *h;
2033         struct field *f;
2034         char *header_name, *field_name;
2035
2036         if ((name[0] != 'h') || (name[1] != '.'))
2037                 return NULL;
2038
2039         header_name = strdup(&name[2]);
2040         if (!header_name)
2041                 return NULL;
2042
2043         field_name = strchr(header_name, '.');
2044         if (!field_name) {
2045                 free(header_name);
2046                 return NULL;
2047         }
2048
2049         *field_name = 0;
2050         field_name++;
2051
2052         h = header_find(p, header_name);
2053         if (!h) {
2054                 free(header_name);
2055                 return NULL;
2056         }
2057
2058         f = struct_type_field_find(h->st, field_name);
2059         if (!f) {
2060                 free(header_name);
2061                 return NULL;
2062         }
2063
2064         if (header)
2065                 *header = h;
2066
2067         free(header_name);
2068         return f;
2069 }
2070
2071 int
2072 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2073                                         const char *name,
2074                                         const char *struct_type_name)
2075 {
2076         struct struct_type *st;
2077         struct header *h;
2078         size_t n_headers_max;
2079
2080         CHECK(p, EINVAL);
2081         CHECK_NAME(name, EINVAL);
2082         CHECK_NAME(struct_type_name, EINVAL);
2083
2084         CHECK(!header_find(p, name), EEXIST);
2085
2086         st = struct_type_find(p, struct_type_name);
2087         CHECK(st, EINVAL);
2088
2089         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2090         CHECK(p->n_headers < n_headers_max, ENOSPC);
2091
2092         /* Node allocation. */
2093         h = calloc(1, sizeof(struct header));
2094         CHECK(h, ENOMEM);
2095
2096         /* Node initialization. */
2097         strcpy(h->name, name);
2098         h->st = st;
2099         h->struct_id = p->n_structs;
2100         h->id = p->n_headers;
2101
2102         /* Node add to tailq. */
2103         TAILQ_INSERT_TAIL(&p->headers, h, node);
2104         p->n_headers++;
2105         p->n_structs++;
2106
2107         return 0;
2108 }
2109
2110 static int
2111 header_build(struct rte_swx_pipeline *p)
2112 {
2113         struct header *h;
2114         uint32_t n_bytes = 0, i;
2115
2116         TAILQ_FOREACH(h, &p->headers, node) {
2117                 n_bytes += h->st->n_bits / 8;
2118         }
2119
2120         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2121                 struct thread *t = &p->threads[i];
2122                 uint32_t offset = 0;
2123
2124                 t->headers = calloc(p->n_headers,
2125                                     sizeof(struct header_runtime));
2126                 CHECK(t->headers, ENOMEM);
2127
2128                 t->headers_out = calloc(p->n_headers,
2129                                         sizeof(struct header_out_runtime));
2130                 CHECK(t->headers_out, ENOMEM);
2131
2132                 t->header_storage = calloc(1, n_bytes);
2133                 CHECK(t->header_storage, ENOMEM);
2134
2135                 t->header_out_storage = calloc(1, n_bytes);
2136                 CHECK(t->header_out_storage, ENOMEM);
2137
2138                 TAILQ_FOREACH(h, &p->headers, node) {
2139                         uint8_t *header_storage;
2140
2141                         header_storage = &t->header_storage[offset];
2142                         offset += h->st->n_bits / 8;
2143
2144                         t->headers[h->id].ptr0 = header_storage;
2145                         t->structs[h->struct_id] = header_storage;
2146                 }
2147         }
2148
2149         return 0;
2150 }
2151
2152 static void
2153 header_build_free(struct rte_swx_pipeline *p)
2154 {
2155         uint32_t i;
2156
2157         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2158                 struct thread *t = &p->threads[i];
2159
2160                 free(t->headers_out);
2161                 t->headers_out = NULL;
2162
2163                 free(t->headers);
2164                 t->headers = NULL;
2165
2166                 free(t->header_out_storage);
2167                 t->header_out_storage = NULL;
2168
2169                 free(t->header_storage);
2170                 t->header_storage = NULL;
2171         }
2172 }
2173
2174 static void
2175 header_free(struct rte_swx_pipeline *p)
2176 {
2177         header_build_free(p);
2178
2179         for ( ; ; ) {
2180                 struct header *elem;
2181
2182                 elem = TAILQ_FIRST(&p->headers);
2183                 if (!elem)
2184                         break;
2185
2186                 TAILQ_REMOVE(&p->headers, elem, node);
2187                 free(elem);
2188         }
2189 }
2190
2191 /*
2192  * Meta-data.
2193  */
2194 static struct field *
2195 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2196 {
2197         if (!p->metadata_st)
2198                 return NULL;
2199
2200         if (name[0] != 'm' || name[1] != '.')
2201                 return NULL;
2202
2203         return struct_type_field_find(p->metadata_st, &name[2]);
2204 }
2205
2206 int
2207 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2208                                           const char *struct_type_name)
2209 {
2210         struct struct_type *st = NULL;
2211
2212         CHECK(p, EINVAL);
2213
2214         CHECK_NAME(struct_type_name, EINVAL);
2215         st  = struct_type_find(p, struct_type_name);
2216         CHECK(st, EINVAL);
2217         CHECK(!p->metadata_st, EINVAL);
2218
2219         p->metadata_st = st;
2220         p->metadata_struct_id = p->n_structs;
2221
2222         p->n_structs++;
2223
2224         return 0;
2225 }
2226
2227 static int
2228 metadata_build(struct rte_swx_pipeline *p)
2229 {
2230         uint32_t n_bytes = p->metadata_st->n_bits / 8;
2231         uint32_t i;
2232
2233         /* Thread-level initialization. */
2234         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2235                 struct thread *t = &p->threads[i];
2236                 uint8_t *metadata;
2237
2238                 metadata = calloc(1, n_bytes);
2239                 CHECK(metadata, ENOMEM);
2240
2241                 t->metadata = metadata;
2242                 t->structs[p->metadata_struct_id] = metadata;
2243         }
2244
2245         return 0;
2246 }
2247
2248 static void
2249 metadata_build_free(struct rte_swx_pipeline *p)
2250 {
2251         uint32_t i;
2252
2253         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2254                 struct thread *t = &p->threads[i];
2255
2256                 free(t->metadata);
2257                 t->metadata = NULL;
2258         }
2259 }
2260
2261 static void
2262 metadata_free(struct rte_swx_pipeline *p)
2263 {
2264         metadata_build_free(p);
2265 }
2266
2267 /*
2268  * Instruction.
2269  */
2270 static int
2271 instruction_is_jmp(struct instruction *instr)
2272 {
2273         switch (instr->type) {
2274         case INSTR_JMP:
2275         case INSTR_JMP_VALID:
2276         case INSTR_JMP_INVALID:
2277         case INSTR_JMP_HIT:
2278         case INSTR_JMP_MISS:
2279         case INSTR_JMP_ACTION_HIT:
2280         case INSTR_JMP_ACTION_MISS:
2281         case INSTR_JMP_EQ:
2282         case INSTR_JMP_EQ_S:
2283         case INSTR_JMP_EQ_I:
2284         case INSTR_JMP_NEQ:
2285         case INSTR_JMP_NEQ_S:
2286         case INSTR_JMP_NEQ_I:
2287         case INSTR_JMP_LT:
2288         case INSTR_JMP_LT_MH:
2289         case INSTR_JMP_LT_HM:
2290         case INSTR_JMP_LT_HH:
2291         case INSTR_JMP_LT_MI:
2292         case INSTR_JMP_LT_HI:
2293         case INSTR_JMP_GT:
2294         case INSTR_JMP_GT_MH:
2295         case INSTR_JMP_GT_HM:
2296         case INSTR_JMP_GT_HH:
2297         case INSTR_JMP_GT_MI:
2298         case INSTR_JMP_GT_HI:
2299                 return 1;
2300
2301         default:
2302                 return 0;
2303         }
2304 }
2305
2306 static struct field *
2307 action_field_parse(struct action *action, const char *name);
2308
2309 static struct field *
2310 struct_field_parse(struct rte_swx_pipeline *p,
2311                    struct action *action,
2312                    const char *name,
2313                    uint32_t *struct_id)
2314 {
2315         struct field *f;
2316
2317         switch (name[0]) {
2318         case 'h':
2319         {
2320                 struct header *header;
2321
2322                 f = header_field_parse(p, name, &header);
2323                 if (!f)
2324                         return NULL;
2325
2326                 *struct_id = header->struct_id;
2327                 return f;
2328         }
2329
2330         case 'm':
2331         {
2332                 f = metadata_field_parse(p, name);
2333                 if (!f)
2334                         return NULL;
2335
2336                 *struct_id = p->metadata_struct_id;
2337                 return f;
2338         }
2339
2340         case 't':
2341         {
2342                 if (!action)
2343                         return NULL;
2344
2345                 f = action_field_parse(action, name);
2346                 if (!f)
2347                         return NULL;
2348
2349                 *struct_id = 0;
2350                 return f;
2351         }
2352
2353         case 'e':
2354         {
2355                 struct extern_obj *obj;
2356
2357                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2358                 if (!f)
2359                         return NULL;
2360
2361                 *struct_id = obj->struct_id;
2362                 return f;
2363         }
2364
2365         case 'f':
2366         {
2367                 struct extern_func *func;
2368
2369                 f = extern_func_mailbox_field_parse(p, name, &func);
2370                 if (!f)
2371                         return NULL;
2372
2373                 *struct_id = func->struct_id;
2374                 return f;
2375         }
2376
2377         default:
2378                 return NULL;
2379         }
2380 }
2381
2382 static inline void
2383 pipeline_port_inc(struct rte_swx_pipeline *p)
2384 {
2385         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2386 }
2387
2388 static inline void
2389 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2390 {
2391         t->ip = p->instructions;
2392 }
2393
2394 static inline void
2395 thread_ip_set(struct thread *t, struct instruction *ip)
2396 {
2397         t->ip = ip;
2398 }
2399
2400 static inline void
2401 thread_ip_action_call(struct rte_swx_pipeline *p,
2402                       struct thread *t,
2403                       uint32_t action_id)
2404 {
2405         t->ret = t->ip + 1;
2406         t->ip = p->action_instructions[action_id];
2407 }
2408
2409 static inline void
2410 thread_ip_inc(struct rte_swx_pipeline *p);
2411
2412 static inline void
2413 thread_ip_inc(struct rte_swx_pipeline *p)
2414 {
2415         struct thread *t = &p->threads[p->thread_id];
2416
2417         t->ip++;
2418 }
2419
2420 static inline void
2421 thread_ip_inc_cond(struct thread *t, int cond)
2422 {
2423         t->ip += cond;
2424 }
2425
2426 static inline void
2427 thread_yield(struct rte_swx_pipeline *p)
2428 {
2429         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2430 }
2431
2432 static inline void
2433 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2434 {
2435         p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2436 }
2437
2438 /*
2439  * rx.
2440  */
2441 static int
2442 instr_rx_translate(struct rte_swx_pipeline *p,
2443                    struct action *action,
2444                    char **tokens,
2445                    int n_tokens,
2446                    struct instruction *instr,
2447                    struct instruction_data *data __rte_unused)
2448 {
2449         struct field *f;
2450
2451         CHECK(!action, EINVAL);
2452         CHECK(n_tokens == 2, EINVAL);
2453
2454         f = metadata_field_parse(p, tokens[1]);
2455         CHECK(f, EINVAL);
2456
2457         instr->type = INSTR_RX;
2458         instr->io.io.offset = f->offset / 8;
2459         instr->io.io.n_bits = f->n_bits;
2460         return 0;
2461 }
2462
2463 static inline void
2464 instr_rx_exec(struct rte_swx_pipeline *p);
2465
2466 static inline void
2467 instr_rx_exec(struct rte_swx_pipeline *p)
2468 {
2469         struct thread *t = &p->threads[p->thread_id];
2470         struct instruction *ip = t->ip;
2471         struct port_in_runtime *port = &p->in[p->port_id];
2472         struct rte_swx_pkt *pkt = &t->pkt;
2473         int pkt_received;
2474
2475         /* Packet. */
2476         pkt_received = port->pkt_rx(port->obj, pkt);
2477         t->ptr = &pkt->pkt[pkt->offset];
2478         rte_prefetch0(t->ptr);
2479
2480         TRACE("[Thread %2u] rx %s from port %u\n",
2481               p->thread_id,
2482               pkt_received ? "1 pkt" : "0 pkts",
2483               p->port_id);
2484
2485         /* Headers. */
2486         t->valid_headers = 0;
2487         t->n_headers_out = 0;
2488
2489         /* Meta-data. */
2490         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2491
2492         /* Tables. */
2493         t->table_state = p->table_state;
2494
2495         /* Thread. */
2496         pipeline_port_inc(p);
2497         thread_ip_inc_cond(t, pkt_received);
2498         thread_yield(p);
2499 }
2500
2501 /*
2502  * tx.
2503  */
2504 static int
2505 instr_tx_translate(struct rte_swx_pipeline *p,
2506                    struct action *action __rte_unused,
2507                    char **tokens,
2508                    int n_tokens,
2509                    struct instruction *instr,
2510                    struct instruction_data *data __rte_unused)
2511 {
2512         struct field *f;
2513
2514         CHECK(n_tokens == 2, EINVAL);
2515
2516         f = metadata_field_parse(p, tokens[1]);
2517         CHECK(f, EINVAL);
2518
2519         instr->type = INSTR_TX;
2520         instr->io.io.offset = f->offset / 8;
2521         instr->io.io.n_bits = f->n_bits;
2522         return 0;
2523 }
2524
2525 static inline void
2526 emit_handler(struct thread *t)
2527 {
2528         struct header_out_runtime *h0 = &t->headers_out[0];
2529         struct header_out_runtime *h1 = &t->headers_out[1];
2530         uint32_t offset = 0, i;
2531
2532         /* No header change or header decapsulation. */
2533         if ((t->n_headers_out == 1) &&
2534             (h0->ptr + h0->n_bytes == t->ptr)) {
2535                 TRACE("Emit handler: no header change or header decap.\n");
2536
2537                 t->pkt.offset -= h0->n_bytes;
2538                 t->pkt.length += h0->n_bytes;
2539
2540                 return;
2541         }
2542
2543         /* Header encapsulation (optionally, with prior header decasulation). */
2544         if ((t->n_headers_out == 2) &&
2545             (h1->ptr + h1->n_bytes == t->ptr) &&
2546             (h0->ptr == h0->ptr0)) {
2547                 uint32_t offset;
2548
2549                 TRACE("Emit handler: header encapsulation.\n");
2550
2551                 offset = h0->n_bytes + h1->n_bytes;
2552                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2553                 t->pkt.offset -= offset;
2554                 t->pkt.length += offset;
2555
2556                 return;
2557         }
2558
2559         /* Header insertion. */
2560         /* TBD */
2561
2562         /* Header extraction. */
2563         /* TBD */
2564
2565         /* For any other case. */
2566         TRACE("Emit handler: complex case.\n");
2567
2568         for (i = 0; i < t->n_headers_out; i++) {
2569                 struct header_out_runtime *h = &t->headers_out[i];
2570
2571                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2572                 offset += h->n_bytes;
2573         }
2574
2575         if (offset) {
2576                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2577                 t->pkt.offset -= offset;
2578                 t->pkt.length += offset;
2579         }
2580 }
2581
2582 static inline void
2583 instr_tx_exec(struct rte_swx_pipeline *p);
2584
2585 static inline void
2586 instr_tx_exec(struct rte_swx_pipeline *p)
2587 {
2588         struct thread *t = &p->threads[p->thread_id];
2589         struct instruction *ip = t->ip;
2590         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2591         struct port_out_runtime *port = &p->out[port_id];
2592         struct rte_swx_pkt *pkt = &t->pkt;
2593
2594         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2595               p->thread_id,
2596               (uint32_t)port_id);
2597
2598         /* Headers. */
2599         emit_handler(t);
2600
2601         /* Packet. */
2602         port->pkt_tx(port->obj, pkt);
2603
2604         /* Thread. */
2605         thread_ip_reset(p, t);
2606         instr_rx_exec(p);
2607 }
2608
2609 /*
2610  * extract.
2611  */
2612 static int
2613 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2614                             struct action *action,
2615                             char **tokens,
2616                             int n_tokens,
2617                             struct instruction *instr,
2618                             struct instruction_data *data __rte_unused)
2619 {
2620         struct header *h;
2621
2622         CHECK(!action, EINVAL);
2623         CHECK(n_tokens == 2, EINVAL);
2624
2625         h = header_parse(p, tokens[1]);
2626         CHECK(h, EINVAL);
2627
2628         instr->type = INSTR_HDR_EXTRACT;
2629         instr->io.hdr.header_id[0] = h->id;
2630         instr->io.hdr.struct_id[0] = h->struct_id;
2631         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2632         return 0;
2633 }
2634
2635 static inline void
2636 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2637
2638 static inline void
2639 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2640 {
2641         struct thread *t = &p->threads[p->thread_id];
2642         struct instruction *ip = t->ip;
2643         uint64_t valid_headers = t->valid_headers;
2644         uint8_t *ptr = t->ptr;
2645         uint32_t offset = t->pkt.offset;
2646         uint32_t length = t->pkt.length;
2647         uint32_t i;
2648
2649         for (i = 0; i < n_extract; i++) {
2650                 uint32_t header_id = ip->io.hdr.header_id[i];
2651                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2652                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2653
2654                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2655                       p->thread_id,
2656                       header_id,
2657                       n_bytes);
2658
2659                 /* Headers. */
2660                 t->structs[struct_id] = ptr;
2661                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2662
2663                 /* Packet. */
2664                 offset += n_bytes;
2665                 length -= n_bytes;
2666                 ptr += n_bytes;
2667         }
2668
2669         /* Headers. */
2670         t->valid_headers = valid_headers;
2671
2672         /* Packet. */
2673         t->pkt.offset = offset;
2674         t->pkt.length = length;
2675         t->ptr = ptr;
2676 }
2677
2678 static inline void
2679 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2680 {
2681         __instr_hdr_extract_exec(p, 1);
2682
2683         /* Thread. */
2684         thread_ip_inc(p);
2685 }
2686
2687 static inline void
2688 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2689 {
2690         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2691               p->thread_id);
2692
2693         __instr_hdr_extract_exec(p, 2);
2694
2695         /* Thread. */
2696         thread_ip_inc(p);
2697 }
2698
2699 static inline void
2700 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2701 {
2702         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2703               p->thread_id);
2704
2705         __instr_hdr_extract_exec(p, 3);
2706
2707         /* Thread. */
2708         thread_ip_inc(p);
2709 }
2710
2711 static inline void
2712 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2713 {
2714         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2715               p->thread_id);
2716
2717         __instr_hdr_extract_exec(p, 4);
2718
2719         /* Thread. */
2720         thread_ip_inc(p);
2721 }
2722
2723 static inline void
2724 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2725 {
2726         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2727               p->thread_id);
2728
2729         __instr_hdr_extract_exec(p, 5);
2730
2731         /* Thread. */
2732         thread_ip_inc(p);
2733 }
2734
2735 static inline void
2736 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2737 {
2738         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2739               p->thread_id);
2740
2741         __instr_hdr_extract_exec(p, 6);
2742
2743         /* Thread. */
2744         thread_ip_inc(p);
2745 }
2746
2747 static inline void
2748 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2749 {
2750         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2751               p->thread_id);
2752
2753         __instr_hdr_extract_exec(p, 7);
2754
2755         /* Thread. */
2756         thread_ip_inc(p);
2757 }
2758
2759 static inline void
2760 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2761 {
2762         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2763               p->thread_id);
2764
2765         __instr_hdr_extract_exec(p, 8);
2766
2767         /* Thread. */
2768         thread_ip_inc(p);
2769 }
2770
2771 /*
2772  * emit.
2773  */
2774 static int
2775 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2776                          struct action *action __rte_unused,
2777                          char **tokens,
2778                          int n_tokens,
2779                          struct instruction *instr,
2780                          struct instruction_data *data __rte_unused)
2781 {
2782         struct header *h;
2783
2784         CHECK(n_tokens == 2, EINVAL);
2785
2786         h = header_parse(p, tokens[1]);
2787         CHECK(h, EINVAL);
2788
2789         instr->type = INSTR_HDR_EMIT;
2790         instr->io.hdr.header_id[0] = h->id;
2791         instr->io.hdr.struct_id[0] = h->struct_id;
2792         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2793         return 0;
2794 }
2795
2796 static inline void
2797 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
2798
2799 static inline void
2800 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
2801 {
2802         struct thread *t = &p->threads[p->thread_id];
2803         struct instruction *ip = t->ip;
2804         uint32_t n_headers_out = t->n_headers_out;
2805         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
2806         uint8_t *ho_ptr = NULL;
2807         uint32_t ho_nbytes = 0, i;
2808
2809         for (i = 0; i < n_emit; i++) {
2810                 uint32_t header_id = ip->io.hdr.header_id[i];
2811                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2812                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2813
2814                 struct header_runtime *hi = &t->headers[header_id];
2815                 uint8_t *hi_ptr = t->structs[struct_id];
2816
2817                 TRACE("[Thread %2u]: emit header %u\n",
2818                       p->thread_id,
2819                       header_id);
2820
2821                 /* Headers. */
2822                 if (!i) {
2823                         if (!t->n_headers_out) {
2824                                 ho = &t->headers_out[0];
2825
2826                                 ho->ptr0 = hi->ptr0;
2827                                 ho->ptr = hi_ptr;
2828
2829                                 ho_ptr = hi_ptr;
2830                                 ho_nbytes = n_bytes;
2831
2832                                 n_headers_out = 1;
2833
2834                                 continue;
2835                         } else {
2836                                 ho_ptr = ho->ptr;
2837                                 ho_nbytes = ho->n_bytes;
2838                         }
2839                 }
2840
2841                 if (ho_ptr + ho_nbytes == hi_ptr) {
2842                         ho_nbytes += n_bytes;
2843                 } else {
2844                         ho->n_bytes = ho_nbytes;
2845
2846                         ho++;
2847                         ho->ptr0 = hi->ptr0;
2848                         ho->ptr = hi_ptr;
2849
2850                         ho_ptr = hi_ptr;
2851                         ho_nbytes = n_bytes;
2852
2853                         n_headers_out++;
2854                 }
2855         }
2856
2857         ho->n_bytes = ho_nbytes;
2858         t->n_headers_out = n_headers_out;
2859 }
2860
2861 static inline void
2862 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2863 {
2864         __instr_hdr_emit_exec(p, 1);
2865
2866         /* Thread. */
2867         thread_ip_inc(p);
2868 }
2869
2870 static inline void
2871 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2872 {
2873         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2874               p->thread_id);
2875
2876         __instr_hdr_emit_exec(p, 1);
2877         instr_tx_exec(p);
2878 }
2879
2880 static inline void
2881 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2882 {
2883         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2884               p->thread_id);
2885
2886         __instr_hdr_emit_exec(p, 2);
2887         instr_tx_exec(p);
2888 }
2889
2890 static inline void
2891 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2892 {
2893         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2894               p->thread_id);
2895
2896         __instr_hdr_emit_exec(p, 3);
2897         instr_tx_exec(p);
2898 }
2899
2900 static inline void
2901 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2902 {
2903         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2904               p->thread_id);
2905
2906         __instr_hdr_emit_exec(p, 4);
2907         instr_tx_exec(p);
2908 }
2909
2910 static inline void
2911 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2912 {
2913         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2914               p->thread_id);
2915
2916         __instr_hdr_emit_exec(p, 5);
2917         instr_tx_exec(p);
2918 }
2919
2920 static inline void
2921 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2922 {
2923         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2924               p->thread_id);
2925
2926         __instr_hdr_emit_exec(p, 6);
2927         instr_tx_exec(p);
2928 }
2929
2930 static inline void
2931 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2932 {
2933         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2934               p->thread_id);
2935
2936         __instr_hdr_emit_exec(p, 7);
2937         instr_tx_exec(p);
2938 }
2939
2940 static inline void
2941 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2942 {
2943         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2944               p->thread_id);
2945
2946         __instr_hdr_emit_exec(p, 8);
2947         instr_tx_exec(p);
2948 }
2949
2950 /*
2951  * validate.
2952  */
2953 static int
2954 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2955                              struct action *action __rte_unused,
2956                              char **tokens,
2957                              int n_tokens,
2958                              struct instruction *instr,
2959                              struct instruction_data *data __rte_unused)
2960 {
2961         struct header *h;
2962
2963         CHECK(n_tokens == 2, EINVAL);
2964
2965         h = header_parse(p, tokens[1]);
2966         CHECK(h, EINVAL);
2967
2968         instr->type = INSTR_HDR_VALIDATE;
2969         instr->valid.header_id = h->id;
2970         return 0;
2971 }
2972
2973 static inline void
2974 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2975 {
2976         struct thread *t = &p->threads[p->thread_id];
2977         struct instruction *ip = t->ip;
2978         uint32_t header_id = ip->valid.header_id;
2979
2980         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2981
2982         /* Headers. */
2983         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2984
2985         /* Thread. */
2986         thread_ip_inc(p);
2987 }
2988
2989 /*
2990  * invalidate.
2991  */
2992 static int
2993 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2994                                struct action *action __rte_unused,
2995                                char **tokens,
2996                                int n_tokens,
2997                                struct instruction *instr,
2998                                struct instruction_data *data __rte_unused)
2999 {
3000         struct header *h;
3001
3002         CHECK(n_tokens == 2, EINVAL);
3003
3004         h = header_parse(p, tokens[1]);
3005         CHECK(h, EINVAL);
3006
3007         instr->type = INSTR_HDR_INVALIDATE;
3008         instr->valid.header_id = h->id;
3009         return 0;
3010 }
3011
3012 static inline void
3013 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3014 {
3015         struct thread *t = &p->threads[p->thread_id];
3016         struct instruction *ip = t->ip;
3017         uint32_t header_id = ip->valid.header_id;
3018
3019         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3020
3021         /* Headers. */
3022         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3023
3024         /* Thread. */
3025         thread_ip_inc(p);
3026 }
3027
3028 /*
3029  * table.
3030  */
3031 static struct table *
3032 table_find(struct rte_swx_pipeline *p, const char *name);
3033
3034 static int
3035 instr_table_translate(struct rte_swx_pipeline *p,
3036                       struct action *action,
3037                       char **tokens,
3038                       int n_tokens,
3039                       struct instruction *instr,
3040                       struct instruction_data *data __rte_unused)
3041 {
3042         struct table *t;
3043
3044         CHECK(!action, EINVAL);
3045         CHECK(n_tokens == 2, EINVAL);
3046
3047         t = table_find(p, tokens[1]);
3048         CHECK(t, EINVAL);
3049
3050         instr->type = INSTR_TABLE;
3051         instr->table.table_id = t->id;
3052         return 0;
3053 }
3054
3055 static inline void
3056 instr_table_exec(struct rte_swx_pipeline *p)
3057 {
3058         struct thread *t = &p->threads[p->thread_id];
3059         struct instruction *ip = t->ip;
3060         uint32_t table_id = ip->table.table_id;
3061         struct rte_swx_table_state *ts = &t->table_state[table_id];
3062         struct table_runtime *table = &t->tables[table_id];
3063         uint64_t action_id;
3064         uint8_t *action_data;
3065         int done, hit;
3066
3067         /* Table. */
3068         done = table->func(ts->obj,
3069                            table->mailbox,
3070                            table->key,
3071                            &action_id,
3072                            &action_data,
3073                            &hit);
3074         if (!done) {
3075                 /* Thread. */
3076                 TRACE("[Thread %2u] table %u (not finalized)\n",
3077                       p->thread_id,
3078                       table_id);
3079
3080                 thread_yield(p);
3081                 return;
3082         }
3083
3084         action_id = hit ? action_id : ts->default_action_id;
3085         action_data = hit ? action_data : ts->default_action_data;
3086
3087         TRACE("[Thread %2u] table %u (%s, action %u)\n",
3088               p->thread_id,
3089               table_id,
3090               hit ? "hit" : "miss",
3091               (uint32_t)action_id);
3092
3093         t->action_id = action_id;
3094         t->structs[0] = action_data;
3095         t->hit = hit;
3096
3097         /* Thread. */
3098         thread_ip_action_call(p, t, action_id);
3099 }
3100
3101 /*
3102  * extern.
3103  */
3104 static int
3105 instr_extern_translate(struct rte_swx_pipeline *p,
3106                        struct action *action __rte_unused,
3107                        char **tokens,
3108                        int n_tokens,
3109                        struct instruction *instr,
3110                        struct instruction_data *data __rte_unused)
3111 {
3112         char *token = tokens[1];
3113
3114         CHECK(n_tokens == 2, EINVAL);
3115
3116         if (token[0] == 'e') {
3117                 struct extern_obj *obj;
3118                 struct extern_type_member_func *func;
3119
3120                 func = extern_obj_member_func_parse(p, token, &obj);
3121                 CHECK(func, EINVAL);
3122
3123                 instr->type = INSTR_EXTERN_OBJ;
3124                 instr->ext_obj.ext_obj_id = obj->id;
3125                 instr->ext_obj.func_id = func->id;
3126
3127                 return 0;
3128         }
3129
3130         if (token[0] == 'f') {
3131                 struct extern_func *func;
3132
3133                 func = extern_func_parse(p, token);
3134                 CHECK(func, EINVAL);
3135
3136                 instr->type = INSTR_EXTERN_FUNC;
3137                 instr->ext_func.ext_func_id = func->id;
3138
3139                 return 0;
3140         }
3141
3142         CHECK(0, EINVAL);
3143 }
3144
3145 static inline void
3146 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3147 {
3148         struct thread *t = &p->threads[p->thread_id];
3149         struct instruction *ip = t->ip;
3150         uint32_t obj_id = ip->ext_obj.ext_obj_id;
3151         uint32_t func_id = ip->ext_obj.func_id;
3152         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3153         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3154
3155         TRACE("[Thread %2u] extern obj %u member func %u\n",
3156               p->thread_id,
3157               obj_id,
3158               func_id);
3159
3160         /* Extern object member function execute. */
3161         uint32_t done = func(obj->obj, obj->mailbox);
3162
3163         /* Thread. */
3164         thread_ip_inc_cond(t, done);
3165         thread_yield_cond(p, done ^ 1);
3166 }
3167
3168 static inline void
3169 instr_extern_func_exec(struct rte_swx_pipeline *p)
3170 {
3171         struct thread *t = &p->threads[p->thread_id];
3172         struct instruction *ip = t->ip;
3173         uint32_t ext_func_id = ip->ext_func.ext_func_id;
3174         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3175         rte_swx_extern_func_t func = ext_func->func;
3176
3177         TRACE("[Thread %2u] extern func %u\n",
3178               p->thread_id,
3179               ext_func_id);
3180
3181         /* Extern function execute. */
3182         uint32_t done = func(ext_func->mailbox);
3183
3184         /* Thread. */
3185         thread_ip_inc_cond(t, done);
3186         thread_yield_cond(p, done ^ 1);
3187 }
3188
3189 /*
3190  * mov.
3191  */
3192 static int
3193 instr_mov_translate(struct rte_swx_pipeline *p,
3194                     struct action *action,
3195                     char **tokens,
3196                     int n_tokens,
3197                     struct instruction *instr,
3198                     struct instruction_data *data __rte_unused)
3199 {
3200         char *dst = tokens[1], *src = tokens[2];
3201         struct field *fdst, *fsrc;
3202         uint32_t dst_struct_id, src_struct_id, src_val;
3203
3204         CHECK(n_tokens == 3, EINVAL);
3205
3206         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3207         CHECK(fdst, EINVAL);
3208
3209         /* MOV or MOV_S. */
3210         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3211         if (fsrc) {
3212                 instr->type = INSTR_MOV;
3213                 if ((dst[0] == 'h' && src[0] != 'h') ||
3214                     (dst[0] != 'h' && src[0] == 'h'))
3215                         instr->type = INSTR_MOV_S;
3216
3217                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3218                 instr->mov.dst.n_bits = fdst->n_bits;
3219                 instr->mov.dst.offset = fdst->offset / 8;
3220                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3221                 instr->mov.src.n_bits = fsrc->n_bits;
3222                 instr->mov.src.offset = fsrc->offset / 8;
3223                 return 0;
3224         }
3225
3226         /* MOV_I. */
3227         src_val = strtoul(src, &src, 0);
3228         CHECK(!src[0], EINVAL);
3229
3230         if (dst[0] == 'h')
3231                 src_val = htonl(src_val);
3232
3233         instr->type = INSTR_MOV_I;
3234         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3235         instr->mov.dst.n_bits = fdst->n_bits;
3236         instr->mov.dst.offset = fdst->offset / 8;
3237         instr->mov.src_val = (uint32_t)src_val;
3238         return 0;
3239 }
3240
3241 static inline void
3242 instr_mov_exec(struct rte_swx_pipeline *p)
3243 {
3244         struct thread *t = &p->threads[p->thread_id];
3245         struct instruction *ip = t->ip;
3246
3247         TRACE("[Thread %2u] mov\n",
3248               p->thread_id);
3249
3250         MOV(t, ip);
3251
3252         /* Thread. */
3253         thread_ip_inc(p);
3254 }
3255
3256 static inline void
3257 instr_mov_s_exec(struct rte_swx_pipeline *p)
3258 {
3259         struct thread *t = &p->threads[p->thread_id];
3260         struct instruction *ip = t->ip;
3261
3262         TRACE("[Thread %2u] mov (s)\n",
3263               p->thread_id);
3264
3265         MOV_S(t, ip);
3266
3267         /* Thread. */
3268         thread_ip_inc(p);
3269 }
3270
3271 static inline void
3272 instr_mov_i_exec(struct rte_swx_pipeline *p)
3273 {
3274         struct thread *t = &p->threads[p->thread_id];
3275         struct instruction *ip = t->ip;
3276
3277         TRACE("[Thread %2u] mov m.f %x\n",
3278               p->thread_id,
3279               ip->mov.src_val);
3280
3281         MOV_I(t, ip);
3282
3283         /* Thread. */
3284         thread_ip_inc(p);
3285 }
3286
3287 /*
3288  * dma.
3289  */
3290 static int
3291 instr_dma_translate(struct rte_swx_pipeline *p,
3292                     struct action *action,
3293                     char **tokens,
3294                     int n_tokens,
3295                     struct instruction *instr,
3296                     struct instruction_data *data __rte_unused)
3297 {
3298         char *dst = tokens[1];
3299         char *src = tokens[2];
3300         struct header *h;
3301         struct field *tf;
3302
3303         CHECK(action, EINVAL);
3304         CHECK(n_tokens == 3, EINVAL);
3305
3306         h = header_parse(p, dst);
3307         CHECK(h, EINVAL);
3308
3309         tf = action_field_parse(action, src);
3310         CHECK(tf, EINVAL);
3311
3312         instr->type = INSTR_DMA_HT;
3313         instr->dma.dst.header_id[0] = h->id;
3314         instr->dma.dst.struct_id[0] = h->struct_id;
3315         instr->dma.n_bytes[0] = h->st->n_bits / 8;
3316         instr->dma.src.offset[0] = tf->offset / 8;
3317
3318         return 0;
3319 }
3320
3321 static inline void
3322 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3323
3324 static inline void
3325 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3326 {
3327         struct thread *t = &p->threads[p->thread_id];
3328         struct instruction *ip = t->ip;
3329         uint8_t *action_data = t->structs[0];
3330         uint64_t valid_headers = t->valid_headers;
3331         uint32_t i;
3332
3333         for (i = 0; i < n_dma; i++) {
3334                 uint32_t header_id = ip->dma.dst.header_id[i];
3335                 uint32_t struct_id = ip->dma.dst.struct_id[i];
3336                 uint32_t offset = ip->dma.src.offset[i];
3337                 uint32_t n_bytes = ip->dma.n_bytes[i];
3338
3339                 struct header_runtime *h = &t->headers[header_id];
3340                 uint8_t *h_ptr0 = h->ptr0;
3341                 uint8_t *h_ptr = t->structs[struct_id];
3342
3343                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3344                         h_ptr : h_ptr0;
3345                 void *src = &action_data[offset];
3346
3347                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3348
3349                 /* Headers. */
3350                 memcpy(dst, src, n_bytes);
3351                 t->structs[struct_id] = dst;
3352                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3353         }
3354
3355         t->valid_headers = valid_headers;
3356 }
3357
3358 static inline void
3359 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3360 {
3361         __instr_dma_ht_exec(p, 1);
3362
3363         /* Thread. */
3364         thread_ip_inc(p);
3365 }
3366
3367 static inline void
3368 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3369 {
3370         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3371               p->thread_id);
3372
3373         __instr_dma_ht_exec(p, 2);
3374
3375         /* Thread. */
3376         thread_ip_inc(p);
3377 }
3378
3379 static inline void
3380 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3381 {
3382         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3383               p->thread_id);
3384
3385         __instr_dma_ht_exec(p, 3);
3386
3387         /* Thread. */
3388         thread_ip_inc(p);
3389 }
3390
3391 static inline void
3392 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3393 {
3394         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3395               p->thread_id);
3396
3397         __instr_dma_ht_exec(p, 4);
3398
3399         /* Thread. */
3400         thread_ip_inc(p);
3401 }
3402
3403 static inline void
3404 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3405 {
3406         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3407               p->thread_id);
3408
3409         __instr_dma_ht_exec(p, 5);
3410
3411         /* Thread. */
3412         thread_ip_inc(p);
3413 }
3414
3415 static inline void
3416 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3417 {
3418         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3419               p->thread_id);
3420
3421         __instr_dma_ht_exec(p, 6);
3422
3423         /* Thread. */
3424         thread_ip_inc(p);
3425 }
3426
3427 static inline void
3428 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3429 {
3430         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3431               p->thread_id);
3432
3433         __instr_dma_ht_exec(p, 7);
3434
3435         /* Thread. */
3436         thread_ip_inc(p);
3437 }
3438
3439 static inline void
3440 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3441 {
3442         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3443               p->thread_id);
3444
3445         __instr_dma_ht_exec(p, 8);
3446
3447         /* Thread. */
3448         thread_ip_inc(p);
3449 }
3450
3451 /*
3452  * alu.
3453  */
3454 static int
3455 instr_alu_add_translate(struct rte_swx_pipeline *p,
3456                         struct action *action,
3457                         char **tokens,
3458                         int n_tokens,
3459                         struct instruction *instr,
3460                         struct instruction_data *data __rte_unused)
3461 {
3462         char *dst = tokens[1], *src = tokens[2];
3463         struct field *fdst, *fsrc;
3464         uint32_t dst_struct_id, src_struct_id, src_val;
3465
3466         CHECK(n_tokens == 3, EINVAL);
3467
3468         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3469         CHECK(fdst, EINVAL);
3470
3471         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3472         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3473         if (fsrc) {
3474                 instr->type = INSTR_ALU_ADD;
3475                 if (dst[0] == 'h' && src[0] == 'm')
3476                         instr->type = INSTR_ALU_ADD_HM;
3477                 if (dst[0] == 'm' && src[0] == 'h')
3478                         instr->type = INSTR_ALU_ADD_MH;
3479                 if (dst[0] == 'h' && src[0] == 'h')
3480                         instr->type = INSTR_ALU_ADD_HH;
3481
3482                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3483                 instr->alu.dst.n_bits = fdst->n_bits;
3484                 instr->alu.dst.offset = fdst->offset / 8;
3485                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3486                 instr->alu.src.n_bits = fsrc->n_bits;
3487                 instr->alu.src.offset = fsrc->offset / 8;
3488                 return 0;
3489         }
3490
3491         /* ADD_MI, ADD_HI. */
3492         src_val = strtoul(src, &src, 0);
3493         CHECK(!src[0], EINVAL);
3494
3495         instr->type = INSTR_ALU_ADD_MI;
3496         if (dst[0] == 'h')
3497                 instr->type = INSTR_ALU_ADD_HI;
3498
3499         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3500         instr->alu.dst.n_bits = fdst->n_bits;
3501         instr->alu.dst.offset = fdst->offset / 8;
3502         instr->alu.src_val = (uint32_t)src_val;
3503         return 0;
3504 }
3505
3506 static int
3507 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3508                         struct action *action,
3509                         char **tokens,
3510                         int n_tokens,
3511                         struct instruction *instr,
3512                         struct instruction_data *data __rte_unused)
3513 {
3514         char *dst = tokens[1], *src = tokens[2];
3515         struct field *fdst, *fsrc;
3516         uint32_t dst_struct_id, src_struct_id, src_val;
3517
3518         CHECK(n_tokens == 3, EINVAL);
3519
3520         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3521         CHECK(fdst, EINVAL);
3522
3523         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3524         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3525         if (fsrc) {
3526                 instr->type = INSTR_ALU_SUB;
3527                 if (dst[0] == 'h' && src[0] == 'm')
3528                         instr->type = INSTR_ALU_SUB_HM;
3529                 if (dst[0] == 'm' && src[0] == 'h')
3530                         instr->type = INSTR_ALU_SUB_MH;
3531                 if (dst[0] == 'h' && src[0] == 'h')
3532                         instr->type = INSTR_ALU_SUB_HH;
3533
3534                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3535                 instr->alu.dst.n_bits = fdst->n_bits;
3536                 instr->alu.dst.offset = fdst->offset / 8;
3537                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3538                 instr->alu.src.n_bits = fsrc->n_bits;
3539                 instr->alu.src.offset = fsrc->offset / 8;
3540                 return 0;
3541         }
3542
3543         /* SUB_MI, SUB_HI. */
3544         src_val = strtoul(src, &src, 0);
3545         CHECK(!src[0], EINVAL);
3546
3547         instr->type = INSTR_ALU_SUB_MI;
3548         if (dst[0] == 'h')
3549                 instr->type = INSTR_ALU_SUB_HI;
3550
3551         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3552         instr->alu.dst.n_bits = fdst->n_bits;
3553         instr->alu.dst.offset = fdst->offset / 8;
3554         instr->alu.src_val = (uint32_t)src_val;
3555         return 0;
3556 }
3557
3558 static int
3559 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3560                           struct action *action __rte_unused,
3561                           char **tokens,
3562                           int n_tokens,
3563                           struct instruction *instr,
3564                           struct instruction_data *data __rte_unused)
3565 {
3566         char *dst = tokens[1], *src = tokens[2];
3567         struct header *hdst, *hsrc;
3568         struct field *fdst, *fsrc;
3569
3570         CHECK(n_tokens == 3, EINVAL);
3571
3572         fdst = header_field_parse(p, dst, &hdst);
3573         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3574
3575         /* CKADD_FIELD. */
3576         fsrc = header_field_parse(p, src, &hsrc);
3577         if (fsrc) {
3578                 instr->type = INSTR_ALU_CKADD_FIELD;
3579                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3580                 instr->alu.dst.n_bits = fdst->n_bits;
3581                 instr->alu.dst.offset = fdst->offset / 8;
3582                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3583                 instr->alu.src.n_bits = fsrc->n_bits;
3584                 instr->alu.src.offset = fsrc->offset / 8;
3585                 return 0;
3586         }
3587
3588         /* CKADD_STRUCT, CKADD_STRUCT20. */
3589         hsrc = header_parse(p, src);
3590         CHECK(hsrc, EINVAL);
3591
3592         instr->type = INSTR_ALU_CKADD_STRUCT;
3593         if ((hsrc->st->n_bits / 8) == 20)
3594                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3595
3596         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3597         instr->alu.dst.n_bits = fdst->n_bits;
3598         instr->alu.dst.offset = fdst->offset / 8;
3599         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3600         instr->alu.src.n_bits = hsrc->st->n_bits;
3601         instr->alu.src.offset = 0; /* Unused. */
3602         return 0;
3603 }
3604
3605 static int
3606 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3607                           struct action *action __rte_unused,
3608                           char **tokens,
3609                           int n_tokens,
3610                           struct instruction *instr,
3611                           struct instruction_data *data __rte_unused)
3612 {
3613         char *dst = tokens[1], *src = tokens[2];
3614         struct header *hdst, *hsrc;
3615         struct field *fdst, *fsrc;
3616
3617         CHECK(n_tokens == 3, EINVAL);
3618
3619         fdst = header_field_parse(p, dst, &hdst);
3620         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3621
3622         fsrc = header_field_parse(p, src, &hsrc);
3623         CHECK(fsrc, EINVAL);
3624
3625         instr->type = INSTR_ALU_CKSUB_FIELD;
3626         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3627         instr->alu.dst.n_bits = fdst->n_bits;
3628         instr->alu.dst.offset = fdst->offset / 8;
3629         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3630         instr->alu.src.n_bits = fsrc->n_bits;
3631         instr->alu.src.offset = fsrc->offset / 8;
3632         return 0;
3633 }
3634
3635 static int
3636 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3637                         struct action *action,
3638                         char **tokens,
3639                         int n_tokens,
3640                         struct instruction *instr,
3641                         struct instruction_data *data __rte_unused)
3642 {
3643         char *dst = tokens[1], *src = tokens[2];
3644         struct field *fdst, *fsrc;
3645         uint32_t dst_struct_id, src_struct_id, src_val;
3646
3647         CHECK(n_tokens == 3, EINVAL);
3648
3649         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3650         CHECK(fdst, EINVAL);
3651
3652         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3653         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3654         if (fsrc) {
3655                 instr->type = INSTR_ALU_SHL;
3656                 if (dst[0] == 'h' && src[0] == 'm')
3657                         instr->type = INSTR_ALU_SHL_HM;
3658                 if (dst[0] == 'm' && src[0] == 'h')
3659                         instr->type = INSTR_ALU_SHL_MH;
3660                 if (dst[0] == 'h' && src[0] == 'h')
3661                         instr->type = INSTR_ALU_SHL_HH;
3662
3663                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3664                 instr->alu.dst.n_bits = fdst->n_bits;
3665                 instr->alu.dst.offset = fdst->offset / 8;
3666                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3667                 instr->alu.src.n_bits = fsrc->n_bits;
3668                 instr->alu.src.offset = fsrc->offset / 8;
3669                 return 0;
3670         }
3671
3672         /* SHL_MI, SHL_HI. */
3673         src_val = strtoul(src, &src, 0);
3674         CHECK(!src[0], EINVAL);
3675
3676         instr->type = INSTR_ALU_SHL_MI;
3677         if (dst[0] == 'h')
3678                 instr->type = INSTR_ALU_SHL_HI;
3679
3680         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3681         instr->alu.dst.n_bits = fdst->n_bits;
3682         instr->alu.dst.offset = fdst->offset / 8;
3683         instr->alu.src_val = (uint32_t)src_val;
3684         return 0;
3685 }
3686
3687 static int
3688 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3689                         struct action *action,
3690                         char **tokens,
3691                         int n_tokens,
3692                         struct instruction *instr,
3693                         struct instruction_data *data __rte_unused)
3694 {
3695         char *dst = tokens[1], *src = tokens[2];
3696         struct field *fdst, *fsrc;
3697         uint32_t dst_struct_id, src_struct_id, src_val;
3698
3699         CHECK(n_tokens == 3, EINVAL);
3700
3701         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3702         CHECK(fdst, EINVAL);
3703
3704         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3705         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3706         if (fsrc) {
3707                 instr->type = INSTR_ALU_SHR;
3708                 if (dst[0] == 'h' && src[0] == 'm')
3709                         instr->type = INSTR_ALU_SHR_HM;
3710                 if (dst[0] == 'm' && src[0] == 'h')
3711                         instr->type = INSTR_ALU_SHR_MH;
3712                 if (dst[0] == 'h' && src[0] == 'h')
3713                         instr->type = INSTR_ALU_SHR_HH;
3714
3715                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3716                 instr->alu.dst.n_bits = fdst->n_bits;
3717                 instr->alu.dst.offset = fdst->offset / 8;
3718                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3719                 instr->alu.src.n_bits = fsrc->n_bits;
3720                 instr->alu.src.offset = fsrc->offset / 8;
3721                 return 0;
3722         }
3723
3724         /* SHR_MI, SHR_HI. */
3725         src_val = strtoul(src, &src, 0);
3726         CHECK(!src[0], EINVAL);
3727
3728         instr->type = INSTR_ALU_SHR_MI;
3729         if (dst[0] == 'h')
3730                 instr->type = INSTR_ALU_SHR_HI;
3731
3732         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3733         instr->alu.dst.n_bits = fdst->n_bits;
3734         instr->alu.dst.offset = fdst->offset / 8;
3735         instr->alu.src_val = (uint32_t)src_val;
3736         return 0;
3737 }
3738
3739 static int
3740 instr_alu_and_translate(struct rte_swx_pipeline *p,
3741                         struct action *action,
3742                         char **tokens,
3743                         int n_tokens,
3744                         struct instruction *instr,
3745                         struct instruction_data *data __rte_unused)
3746 {
3747         char *dst = tokens[1], *src = tokens[2];
3748         struct field *fdst, *fsrc;
3749         uint32_t dst_struct_id, src_struct_id, src_val;
3750
3751         CHECK(n_tokens == 3, EINVAL);
3752
3753         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3754         CHECK(fdst, EINVAL);
3755
3756         /* AND or AND_S. */
3757         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3758         if (fsrc) {
3759                 instr->type = INSTR_ALU_AND;
3760                 if ((dst[0] == 'h' && src[0] != 'h') ||
3761                     (dst[0] != 'h' && src[0] == 'h'))
3762                         instr->type = INSTR_ALU_AND_S;
3763
3764                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3765                 instr->alu.dst.n_bits = fdst->n_bits;
3766                 instr->alu.dst.offset = fdst->offset / 8;
3767                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3768                 instr->alu.src.n_bits = fsrc->n_bits;
3769                 instr->alu.src.offset = fsrc->offset / 8;
3770                 return 0;
3771         }
3772
3773         /* AND_I. */
3774         src_val = strtoul(src, &src, 0);
3775         CHECK(!src[0], EINVAL);
3776
3777         if (dst[0] == 'h')
3778                 src_val = htonl(src_val);
3779
3780         instr->type = INSTR_ALU_AND_I;
3781         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3782         instr->alu.dst.n_bits = fdst->n_bits;
3783         instr->alu.dst.offset = fdst->offset / 8;
3784         instr->alu.src_val = (uint32_t)src_val;
3785         return 0;
3786 }
3787
3788 static int
3789 instr_alu_or_translate(struct rte_swx_pipeline *p,
3790                        struct action *action,
3791                        char **tokens,
3792                        int n_tokens,
3793                        struct instruction *instr,
3794                        struct instruction_data *data __rte_unused)
3795 {
3796         char *dst = tokens[1], *src = tokens[2];
3797         struct field *fdst, *fsrc;
3798         uint32_t dst_struct_id, src_struct_id, src_val;
3799
3800         CHECK(n_tokens == 3, EINVAL);
3801
3802         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3803         CHECK(fdst, EINVAL);
3804
3805         /* OR or OR_S. */
3806         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3807         if (fsrc) {
3808                 instr->type = INSTR_ALU_OR;
3809                 if ((dst[0] == 'h' && src[0] != 'h') ||
3810                     (dst[0] != 'h' && src[0] == 'h'))
3811                         instr->type = INSTR_ALU_OR_S;
3812
3813                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3814                 instr->alu.dst.n_bits = fdst->n_bits;
3815                 instr->alu.dst.offset = fdst->offset / 8;
3816                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3817                 instr->alu.src.n_bits = fsrc->n_bits;
3818                 instr->alu.src.offset = fsrc->offset / 8;
3819                 return 0;
3820         }
3821
3822         /* OR_I. */
3823         src_val = strtoul(src, &src, 0);
3824         CHECK(!src[0], EINVAL);
3825
3826         if (dst[0] == 'h')
3827                 src_val = htonl(src_val);
3828
3829         instr->type = INSTR_ALU_OR_I;
3830         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3831         instr->alu.dst.n_bits = fdst->n_bits;
3832         instr->alu.dst.offset = fdst->offset / 8;
3833         instr->alu.src_val = (uint32_t)src_val;
3834         return 0;
3835 }
3836
3837 static int
3838 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3839                         struct action *action,
3840                         char **tokens,
3841                         int n_tokens,
3842                         struct instruction *instr,
3843                         struct instruction_data *data __rte_unused)
3844 {
3845         char *dst = tokens[1], *src = tokens[2];
3846         struct field *fdst, *fsrc;
3847         uint32_t dst_struct_id, src_struct_id, src_val;
3848
3849         CHECK(n_tokens == 3, EINVAL);
3850
3851         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3852         CHECK(fdst, EINVAL);
3853
3854         /* XOR or XOR_S. */
3855         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3856         if (fsrc) {
3857                 instr->type = INSTR_ALU_XOR;
3858                 if ((dst[0] == 'h' && src[0] != 'h') ||
3859                     (dst[0] != 'h' && src[0] == 'h'))
3860                         instr->type = INSTR_ALU_XOR_S;
3861
3862                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3863                 instr->alu.dst.n_bits = fdst->n_bits;
3864                 instr->alu.dst.offset = fdst->offset / 8;
3865                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3866                 instr->alu.src.n_bits = fsrc->n_bits;
3867                 instr->alu.src.offset = fsrc->offset / 8;
3868                 return 0;
3869         }
3870
3871         /* XOR_I. */
3872         src_val = strtoul(src, &src, 0);
3873         CHECK(!src[0], EINVAL);
3874
3875         if (dst[0] == 'h')
3876                 src_val = htonl(src_val);
3877
3878         instr->type = INSTR_ALU_XOR_I;
3879         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3880         instr->alu.dst.n_bits = fdst->n_bits;
3881         instr->alu.dst.offset = fdst->offset / 8;
3882         instr->alu.src_val = (uint32_t)src_val;
3883         return 0;
3884 }
3885
3886 static inline void
3887 instr_alu_add_exec(struct rte_swx_pipeline *p)
3888 {
3889         struct thread *t = &p->threads[p->thread_id];
3890         struct instruction *ip = t->ip;
3891
3892         TRACE("[Thread %2u] add\n", p->thread_id);
3893
3894         /* Structs. */
3895         ALU(t, ip, +);
3896
3897         /* Thread. */
3898         thread_ip_inc(p);
3899 }
3900
3901 static inline void
3902 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3903 {
3904         struct thread *t = &p->threads[p->thread_id];
3905         struct instruction *ip = t->ip;
3906
3907         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3908
3909         /* Structs. */
3910         ALU_MH(t, ip, +);
3911
3912         /* Thread. */
3913         thread_ip_inc(p);
3914 }
3915
3916 static inline void
3917 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3918 {
3919         struct thread *t = &p->threads[p->thread_id];
3920         struct instruction *ip = t->ip;
3921
3922         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3923
3924         /* Structs. */
3925         ALU_HM(t, ip, +);
3926
3927         /* Thread. */
3928         thread_ip_inc(p);
3929 }
3930
3931 static inline void
3932 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3933 {
3934         struct thread *t = &p->threads[p->thread_id];
3935         struct instruction *ip = t->ip;
3936
3937         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3938
3939         /* Structs. */
3940         ALU_HH(t, ip, +);
3941
3942         /* Thread. */
3943         thread_ip_inc(p);
3944 }
3945
3946 static inline void
3947 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3948 {
3949         struct thread *t = &p->threads[p->thread_id];
3950         struct instruction *ip = t->ip;
3951
3952         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3953
3954         /* Structs. */
3955         ALU_MI(t, ip, +);
3956
3957         /* Thread. */
3958         thread_ip_inc(p);
3959 }
3960
3961 static inline void
3962 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3963 {
3964         struct thread *t = &p->threads[p->thread_id];
3965         struct instruction *ip = t->ip;
3966
3967         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3968
3969         /* Structs. */
3970         ALU_HI(t, ip, +);
3971
3972         /* Thread. */
3973         thread_ip_inc(p);
3974 }
3975
3976 static inline void
3977 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3978 {
3979         struct thread *t = &p->threads[p->thread_id];
3980         struct instruction *ip = t->ip;
3981
3982         TRACE("[Thread %2u] sub\n", p->thread_id);
3983
3984         /* Structs. */
3985         ALU(t, ip, -);
3986
3987         /* Thread. */
3988         thread_ip_inc(p);
3989 }
3990
3991 static inline void
3992 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3993 {
3994         struct thread *t = &p->threads[p->thread_id];
3995         struct instruction *ip = t->ip;
3996
3997         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3998
3999         /* Structs. */
4000         ALU_MH(t, ip, -);
4001
4002         /* Thread. */
4003         thread_ip_inc(p);
4004 }
4005
4006 static inline void
4007 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4008 {
4009         struct thread *t = &p->threads[p->thread_id];
4010         struct instruction *ip = t->ip;
4011
4012         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4013
4014         /* Structs. */
4015         ALU_HM(t, ip, -);
4016
4017         /* Thread. */
4018         thread_ip_inc(p);
4019 }
4020
4021 static inline void
4022 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4023 {
4024         struct thread *t = &p->threads[p->thread_id];
4025         struct instruction *ip = t->ip;
4026
4027         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4028
4029         /* Structs. */
4030         ALU_HH(t, ip, -);
4031
4032         /* Thread. */
4033         thread_ip_inc(p);
4034 }
4035
4036 static inline void
4037 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4038 {
4039         struct thread *t = &p->threads[p->thread_id];
4040         struct instruction *ip = t->ip;
4041
4042         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4043
4044         /* Structs. */
4045         ALU_MI(t, ip, -);
4046
4047         /* Thread. */
4048         thread_ip_inc(p);
4049 }
4050
4051 static inline void
4052 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4053 {
4054         struct thread *t = &p->threads[p->thread_id];
4055         struct instruction *ip = t->ip;
4056
4057         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4058
4059         /* Structs. */
4060         ALU_HI(t, ip, -);
4061
4062         /* Thread. */
4063         thread_ip_inc(p);
4064 }
4065
4066 static inline void
4067 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4068 {
4069         struct thread *t = &p->threads[p->thread_id];
4070         struct instruction *ip = t->ip;
4071
4072         TRACE("[Thread %2u] shl\n", p->thread_id);
4073
4074         /* Structs. */
4075         ALU(t, ip, <<);
4076
4077         /* Thread. */
4078         thread_ip_inc(p);
4079 }
4080
4081 static inline void
4082 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4083 {
4084         struct thread *t = &p->threads[p->thread_id];
4085         struct instruction *ip = t->ip;
4086
4087         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4088
4089         /* Structs. */
4090         ALU_MH(t, ip, <<);
4091
4092         /* Thread. */
4093         thread_ip_inc(p);
4094 }
4095
4096 static inline void
4097 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4098 {
4099         struct thread *t = &p->threads[p->thread_id];
4100         struct instruction *ip = t->ip;
4101
4102         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4103
4104         /* Structs. */
4105         ALU_HM(t, ip, <<);
4106
4107         /* Thread. */
4108         thread_ip_inc(p);
4109 }
4110
4111 static inline void
4112 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4113 {
4114         struct thread *t = &p->threads[p->thread_id];
4115         struct instruction *ip = t->ip;
4116
4117         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4118
4119         /* Structs. */
4120         ALU_HH(t, ip, <<);
4121
4122         /* Thread. */
4123         thread_ip_inc(p);
4124 }
4125
4126 static inline void
4127 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4128 {
4129         struct thread *t = &p->threads[p->thread_id];
4130         struct instruction *ip = t->ip;
4131
4132         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4133
4134         /* Structs. */
4135         ALU_MI(t, ip, <<);
4136
4137         /* Thread. */
4138         thread_ip_inc(p);
4139 }
4140
4141 static inline void
4142 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4143 {
4144         struct thread *t = &p->threads[p->thread_id];
4145         struct instruction *ip = t->ip;
4146
4147         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4148
4149         /* Structs. */
4150         ALU_HI(t, ip, <<);
4151
4152         /* Thread. */
4153         thread_ip_inc(p);
4154 }
4155
4156 static inline void
4157 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4158 {
4159         struct thread *t = &p->threads[p->thread_id];
4160         struct instruction *ip = t->ip;
4161
4162         TRACE("[Thread %2u] shr\n", p->thread_id);
4163
4164         /* Structs. */
4165         ALU(t, ip, >>);
4166
4167         /* Thread. */
4168         thread_ip_inc(p);
4169 }
4170
4171 static inline void
4172 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4173 {
4174         struct thread *t = &p->threads[p->thread_id];
4175         struct instruction *ip = t->ip;
4176
4177         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4178
4179         /* Structs. */
4180         ALU_MH(t, ip, >>);
4181
4182         /* Thread. */
4183         thread_ip_inc(p);
4184 }
4185
4186 static inline void
4187 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4188 {
4189         struct thread *t = &p->threads[p->thread_id];
4190         struct instruction *ip = t->ip;
4191
4192         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4193
4194         /* Structs. */
4195         ALU_HM(t, ip, >>);
4196
4197         /* Thread. */
4198         thread_ip_inc(p);
4199 }
4200
4201 static inline void
4202 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4203 {
4204         struct thread *t = &p->threads[p->thread_id];
4205         struct instruction *ip = t->ip;
4206
4207         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4208
4209         /* Structs. */
4210         ALU_HH(t, ip, >>);
4211
4212         /* Thread. */
4213         thread_ip_inc(p);
4214 }
4215
4216 static inline void
4217 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4218 {
4219         struct thread *t = &p->threads[p->thread_id];
4220         struct instruction *ip = t->ip;
4221
4222         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4223
4224         /* Structs. */
4225         ALU_MI(t, ip, >>);
4226
4227         /* Thread. */
4228         thread_ip_inc(p);
4229 }
4230
4231 static inline void
4232 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4233 {
4234         struct thread *t = &p->threads[p->thread_id];
4235         struct instruction *ip = t->ip;
4236
4237         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4238
4239         /* Structs. */
4240         ALU_HI(t, ip, >>);
4241
4242         /* Thread. */
4243         thread_ip_inc(p);
4244 }
4245
4246 static inline void
4247 instr_alu_and_exec(struct rte_swx_pipeline *p)
4248 {
4249         struct thread *t = &p->threads[p->thread_id];
4250         struct instruction *ip = t->ip;
4251
4252         TRACE("[Thread %2u] and\n", p->thread_id);
4253
4254         /* Structs. */
4255         ALU(t, ip, &);
4256
4257         /* Thread. */
4258         thread_ip_inc(p);
4259 }
4260
4261 static inline void
4262 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
4263 {
4264         struct thread *t = &p->threads[p->thread_id];
4265         struct instruction *ip = t->ip;
4266
4267         TRACE("[Thread %2u] and (s)\n", p->thread_id);
4268
4269         /* Structs. */
4270         ALU_S(t, ip, &);
4271
4272         /* Thread. */
4273         thread_ip_inc(p);
4274 }
4275
4276 static inline void
4277 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4278 {
4279         struct thread *t = &p->threads[p->thread_id];
4280         struct instruction *ip = t->ip;
4281
4282         TRACE("[Thread %2u] and (i)\n", p->thread_id);
4283
4284         /* Structs. */
4285         ALU_I(t, ip, &);
4286
4287         /* Thread. */
4288         thread_ip_inc(p);
4289 }
4290
4291 static inline void
4292 instr_alu_or_exec(struct rte_swx_pipeline *p)
4293 {
4294         struct thread *t = &p->threads[p->thread_id];
4295         struct instruction *ip = t->ip;
4296
4297         TRACE("[Thread %2u] or\n", p->thread_id);
4298
4299         /* Structs. */
4300         ALU(t, ip, |);
4301
4302         /* Thread. */
4303         thread_ip_inc(p);
4304 }
4305
4306 static inline void
4307 instr_alu_or_s_exec(struct rte_swx_pipeline *p)
4308 {
4309         struct thread *t = &p->threads[p->thread_id];
4310         struct instruction *ip = t->ip;
4311
4312         TRACE("[Thread %2u] or (s)\n", p->thread_id);
4313
4314         /* Structs. */
4315         ALU_S(t, ip, |);
4316
4317         /* Thread. */
4318         thread_ip_inc(p);
4319 }
4320
4321 static inline void
4322 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4323 {
4324         struct thread *t = &p->threads[p->thread_id];
4325         struct instruction *ip = t->ip;
4326
4327         TRACE("[Thread %2u] or (i)\n", p->thread_id);
4328
4329         /* Structs. */
4330         ALU_I(t, ip, |);
4331
4332         /* Thread. */
4333         thread_ip_inc(p);
4334 }
4335
4336 static inline void
4337 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4338 {
4339         struct thread *t = &p->threads[p->thread_id];
4340         struct instruction *ip = t->ip;
4341
4342         TRACE("[Thread %2u] xor\n", p->thread_id);
4343
4344         /* Structs. */
4345         ALU(t, ip, ^);
4346
4347         /* Thread. */
4348         thread_ip_inc(p);
4349 }
4350
4351 static inline void
4352 instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
4353 {
4354         struct thread *t = &p->threads[p->thread_id];
4355         struct instruction *ip = t->ip;
4356
4357         TRACE("[Thread %2u] xor (s)\n", p->thread_id);
4358
4359         /* Structs. */
4360         ALU_S(t, ip, ^);
4361
4362         /* Thread. */
4363         thread_ip_inc(p);
4364 }
4365
4366 static inline void
4367 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4368 {
4369         struct thread *t = &p->threads[p->thread_id];
4370         struct instruction *ip = t->ip;
4371
4372         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
4373
4374         /* Structs. */
4375         ALU_I(t, ip, ^);
4376
4377         /* Thread. */
4378         thread_ip_inc(p);
4379 }
4380
4381 static inline void
4382 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4383 {
4384         struct thread *t = &p->threads[p->thread_id];
4385         struct instruction *ip = t->ip;
4386         uint8_t *dst_struct, *src_struct;
4387         uint16_t *dst16_ptr, dst;
4388         uint64_t *src64_ptr, src64, src64_mask, src;
4389         uint64_t r;
4390
4391         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
4392
4393         /* Structs. */
4394         dst_struct = t->structs[ip->alu.dst.struct_id];
4395         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4396         dst = *dst16_ptr;
4397
4398         src_struct = t->structs[ip->alu.src.struct_id];
4399         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4400         src64 = *src64_ptr;
4401         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4402         src = src64 & src64_mask;
4403
4404         r = dst;
4405         r = ~r & 0xFFFF;
4406
4407         /* The first input (r) is a 16-bit number. The second and the third
4408          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
4409          * three numbers (output r) is a 34-bit number.
4410          */
4411         r += (src >> 32) + (src & 0xFFFFFFFF);
4412
4413         /* The first input is a 16-bit number. The second input is an 18-bit
4414          * number. In the worst case scenario, the sum of the two numbers is a
4415          * 19-bit number.
4416          */
4417         r = (r & 0xFFFF) + (r >> 16);
4418
4419         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4420          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
4421          */
4422         r = (r & 0xFFFF) + (r >> 16);
4423
4424         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4425          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4426          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
4427          * therefore the output r is always a 16-bit number.
4428          */
4429         r = (r & 0xFFFF) + (r >> 16);
4430
4431         r = ~r & 0xFFFF;
4432         r = r ? r : 0xFFFF;
4433
4434         *dst16_ptr = (uint16_t)r;
4435
4436         /* Thread. */
4437         thread_ip_inc(p);
4438 }
4439
4440 static inline void
4441 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4442 {
4443         struct thread *t = &p->threads[p->thread_id];
4444         struct instruction *ip = t->ip;
4445         uint8_t *dst_struct, *src_struct;
4446         uint16_t *dst16_ptr, dst;
4447         uint64_t *src64_ptr, src64, src64_mask, src;
4448         uint64_t r;
4449
4450         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
4451
4452         /* Structs. */
4453         dst_struct = t->structs[ip->alu.dst.struct_id];
4454         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4455         dst = *dst16_ptr;
4456
4457         src_struct = t->structs[ip->alu.src.struct_id];
4458         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4459         src64 = *src64_ptr;
4460         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4461         src = src64 & src64_mask;
4462
4463         r = dst;
4464         r = ~r & 0xFFFF;
4465
4466         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
4467          * the following sequence of operations in 2's complement arithmetic:
4468          *    a '- b = (a - b) % 0xFFFF.
4469          *
4470          * In order to prevent an underflow for the below subtraction, in which
4471          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
4472          * minuend), we first add a multiple of the 0xFFFF modulus to the
4473          * minuend. The number we add to the minuend needs to be a 34-bit number
4474          * or higher, so for readability reasons we picked the 36-bit multiple.
4475          * We are effectively turning the 16-bit minuend into a 36-bit number:
4476          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
4477          */
4478         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
4479
4480         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
4481          * result (the output r) is a 36-bit number.
4482          */
4483         r -= (src >> 32) + (src & 0xFFFFFFFF);
4484
4485         /* The first input is a 16-bit number. The second input is a 20-bit
4486          * number. Their sum is a 21-bit number.
4487          */
4488         r = (r & 0xFFFF) + (r >> 16);
4489
4490         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4491          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
4492          */
4493         r = (r & 0xFFFF) + (r >> 16);
4494
4495         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4496          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4497          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4498          * generated, therefore the output r is always a 16-bit number.
4499          */
4500         r = (r & 0xFFFF) + (r >> 16);
4501
4502         r = ~r & 0xFFFF;
4503         r = r ? r : 0xFFFF;
4504
4505         *dst16_ptr = (uint16_t)r;
4506
4507         /* Thread. */
4508         thread_ip_inc(p);
4509 }
4510
4511 static inline void
4512 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4513 {
4514         struct thread *t = &p->threads[p->thread_id];
4515         struct instruction *ip = t->ip;
4516         uint8_t *dst_struct, *src_struct;
4517         uint16_t *dst16_ptr;
4518         uint32_t *src32_ptr;
4519         uint64_t r0, r1;
4520
4521         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
4522
4523         /* Structs. */
4524         dst_struct = t->structs[ip->alu.dst.struct_id];
4525         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4526
4527         src_struct = t->structs[ip->alu.src.struct_id];
4528         src32_ptr = (uint32_t *)&src_struct[0];
4529
4530         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
4531         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
4532         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
4533         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
4534         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
4535
4536         /* The first input is a 16-bit number. The second input is a 19-bit
4537          * number. Their sum is a 20-bit number.
4538          */
4539         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4540
4541         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4542          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
4543          */
4544         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4545
4546         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4547          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4548          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
4549          * generated, therefore the output r is always a 16-bit number.
4550          */
4551         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4552
4553         r0 = ~r0 & 0xFFFF;
4554         r0 = r0 ? r0 : 0xFFFF;
4555
4556         *dst16_ptr = (uint16_t)r0;
4557
4558         /* Thread. */
4559         thread_ip_inc(p);
4560 }
4561
4562 static inline void
4563 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4564 {
4565         struct thread *t = &p->threads[p->thread_id];
4566         struct instruction *ip = t->ip;
4567         uint8_t *dst_struct, *src_struct;
4568         uint16_t *dst16_ptr;
4569         uint32_t *src32_ptr;
4570         uint64_t r = 0;
4571         uint32_t i;
4572
4573         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
4574
4575         /* Structs. */
4576         dst_struct = t->structs[ip->alu.dst.struct_id];
4577         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4578
4579         src_struct = t->structs[ip->alu.src.struct_id];
4580         src32_ptr = (uint32_t *)&src_struct[0];
4581
4582         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
4583          * Therefore, in the worst case scenario, a 35-bit number is added to a
4584          * 16-bit number (the input r), so the output r is 36-bit number.
4585          */
4586         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
4587                 r += *src32_ptr;
4588
4589         /* The first input is a 16-bit number. The second input is a 20-bit
4590          * number. Their sum is a 21-bit number.
4591          */
4592         r = (r & 0xFFFF) + (r >> 16);
4593
4594         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4595          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
4596          */
4597         r = (r & 0xFFFF) + (r >> 16);
4598
4599         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4600          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4601          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4602          * generated, therefore the output r is always a 16-bit number.
4603          */
4604         r = (r & 0xFFFF) + (r >> 16);
4605
4606         r = ~r & 0xFFFF;
4607         r = r ? r : 0xFFFF;
4608
4609         *dst16_ptr = (uint16_t)r;
4610
4611         /* Thread. */
4612         thread_ip_inc(p);
4613 }
4614
4615 /*
4616  * jmp.
4617  */
4618 static struct action *
4619 action_find(struct rte_swx_pipeline *p, const char *name);
4620
4621 static int
4622 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4623                     struct action *action __rte_unused,
4624                     char **tokens,
4625                     int n_tokens,
4626                     struct instruction *instr,
4627                     struct instruction_data *data)
4628 {
4629         CHECK(n_tokens == 2, EINVAL);
4630
4631         strcpy(data->jmp_label, tokens[1]);
4632
4633         instr->type = INSTR_JMP;
4634         instr->jmp.ip = NULL; /* Resolved later. */
4635         return 0;
4636 }
4637
4638 static int
4639 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4640                           struct action *action __rte_unused,
4641                           char **tokens,
4642                           int n_tokens,
4643                           struct instruction *instr,
4644                           struct instruction_data *data)
4645 {
4646         struct header *h;
4647
4648         CHECK(n_tokens == 3, EINVAL);
4649
4650         strcpy(data->jmp_label, tokens[1]);
4651
4652         h = header_parse(p, tokens[2]);
4653         CHECK(h, EINVAL);
4654
4655         instr->type = INSTR_JMP_VALID;
4656         instr->jmp.ip = NULL; /* Resolved later. */
4657         instr->jmp.header_id = h->id;
4658         return 0;
4659 }
4660
4661 static int
4662 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4663                             struct action *action __rte_unused,
4664                             char **tokens,
4665                             int n_tokens,
4666                             struct instruction *instr,
4667                             struct instruction_data *data)
4668 {
4669         struct header *h;
4670
4671         CHECK(n_tokens == 3, EINVAL);
4672
4673         strcpy(data->jmp_label, tokens[1]);
4674
4675         h = header_parse(p, tokens[2]);
4676         CHECK(h, EINVAL);
4677
4678         instr->type = INSTR_JMP_INVALID;
4679         instr->jmp.ip = NULL; /* Resolved later. */
4680         instr->jmp.header_id = h->id;
4681         return 0;
4682 }
4683
4684 static int
4685 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4686                         struct action *action,
4687                         char **tokens,
4688                         int n_tokens,
4689                         struct instruction *instr,
4690                         struct instruction_data *data)
4691 {
4692         CHECK(!action, EINVAL);
4693         CHECK(n_tokens == 2, EINVAL);
4694
4695         strcpy(data->jmp_label, tokens[1]);
4696
4697         instr->type = INSTR_JMP_HIT;
4698         instr->jmp.ip = NULL; /* Resolved later. */
4699         return 0;
4700 }
4701
4702 static int
4703 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4704                          struct action *action,
4705                          char **tokens,
4706                          int n_tokens,
4707                          struct instruction *instr,
4708                          struct instruction_data *data)
4709 {
4710         CHECK(!action, EINVAL);
4711         CHECK(n_tokens == 2, EINVAL);
4712
4713         strcpy(data->jmp_label, tokens[1]);
4714
4715         instr->type = INSTR_JMP_MISS;
4716         instr->jmp.ip = NULL; /* Resolved later. */
4717         return 0;
4718 }
4719
4720 static int
4721 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4722                                struct action *action,
4723                                char **tokens,
4724                                int n_tokens,
4725                                struct instruction *instr,
4726                                struct instruction_data *data)
4727 {
4728         struct action *a;
4729
4730         CHECK(!action, EINVAL);
4731         CHECK(n_tokens == 3, EINVAL);
4732
4733         strcpy(data->jmp_label, tokens[1]);
4734
4735         a = action_find(p, tokens[2]);
4736         CHECK(a, EINVAL);
4737
4738         instr->type = INSTR_JMP_ACTION_HIT;
4739         instr->jmp.ip = NULL; /* Resolved later. */
4740         instr->jmp.action_id = a->id;
4741         return 0;
4742 }
4743
4744 static int
4745 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4746                                 struct action *action,
4747                                 char **tokens,
4748                                 int n_tokens,
4749                                 struct instruction *instr,
4750                                 struct instruction_data *data)
4751 {
4752         struct action *a;
4753
4754         CHECK(!action, EINVAL);
4755         CHECK(n_tokens == 3, EINVAL);
4756
4757         strcpy(data->jmp_label, tokens[1]);
4758
4759         a = action_find(p, tokens[2]);
4760         CHECK(a, EINVAL);
4761
4762         instr->type = INSTR_JMP_ACTION_MISS;
4763         instr->jmp.ip = NULL; /* Resolved later. */
4764         instr->jmp.action_id = a->id;
4765         return 0;
4766 }
4767
4768 static int
4769 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
4770                        struct action *action,
4771                        char **tokens,
4772                        int n_tokens,
4773                        struct instruction *instr,
4774                        struct instruction_data *data)
4775 {
4776         char *a = tokens[2], *b = tokens[3];
4777         struct field *fa, *fb;
4778         uint32_t a_struct_id, b_struct_id, b_val;
4779
4780         CHECK(n_tokens == 4, EINVAL);
4781
4782         strcpy(data->jmp_label, tokens[1]);
4783
4784         fa = struct_field_parse(p, action, a, &a_struct_id);
4785         CHECK(fa, EINVAL);
4786
4787         /* JMP_EQ or JMP_EQ_S. */
4788         fb = struct_field_parse(p, action, b, &b_struct_id);
4789         if (fb) {
4790                 instr->type = INSTR_JMP_EQ;
4791                 if ((a[0] == 'h' && b[0] != 'h') ||
4792                     (a[0] != 'h' && b[0] == 'h'))
4793                         instr->type = INSTR_JMP_EQ_S;
4794                 instr->jmp.ip = NULL; /* Resolved later. */
4795
4796                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4797                 instr->jmp.a.n_bits = fa->n_bits;
4798                 instr->jmp.a.offset = fa->offset / 8;
4799                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4800                 instr->jmp.b.n_bits = fb->n_bits;
4801                 instr->jmp.b.offset = fb->offset / 8;
4802                 return 0;
4803         }
4804
4805         /* JMP_EQ_I. */
4806         b_val = strtoul(b, &b, 0);
4807         CHECK(!b[0], EINVAL);
4808
4809         if (a[0] == 'h')
4810                 b_val = htonl(b_val);
4811
4812         instr->type = INSTR_JMP_EQ_I;
4813         instr->jmp.ip = NULL; /* Resolved later. */
4814         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4815         instr->jmp.a.n_bits = fa->n_bits;
4816         instr->jmp.a.offset = fa->offset / 8;
4817         instr->jmp.b_val = (uint32_t)b_val;
4818         return 0;
4819 }
4820
4821 static int
4822 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
4823                         struct action *action,
4824                         char **tokens,
4825                         int n_tokens,
4826                         struct instruction *instr,
4827                         struct instruction_data *data)
4828 {
4829         char *a = tokens[2], *b = tokens[3];
4830         struct field *fa, *fb;
4831         uint32_t a_struct_id, b_struct_id, b_val;
4832
4833         CHECK(n_tokens == 4, EINVAL);
4834
4835         strcpy(data->jmp_label, tokens[1]);
4836
4837         fa = struct_field_parse(p, action, a, &a_struct_id);
4838         CHECK(fa, EINVAL);
4839
4840         /* JMP_NEQ or JMP_NEQ_S. */
4841         fb = struct_field_parse(p, action, b, &b_struct_id);
4842         if (fb) {
4843                 instr->type = INSTR_JMP_NEQ;
4844                 if ((a[0] == 'h' && b[0] != 'h') ||
4845                     (a[0] != 'h' && b[0] == 'h'))
4846                         instr->type = INSTR_JMP_NEQ_S;
4847                 instr->jmp.ip = NULL; /* Resolved later. */
4848
4849                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4850                 instr->jmp.a.n_bits = fa->n_bits;
4851                 instr->jmp.a.offset = fa->offset / 8;
4852                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4853                 instr->jmp.b.n_bits = fb->n_bits;
4854                 instr->jmp.b.offset = fb->offset / 8;
4855                 return 0;
4856         }
4857
4858         /* JMP_NEQ_I. */
4859         b_val = strtoul(b, &b, 0);
4860         CHECK(!b[0], EINVAL);
4861
4862         if (a[0] == 'h')
4863                 b_val = htonl(b_val);
4864
4865         instr->type = INSTR_JMP_NEQ_I;
4866         instr->jmp.ip = NULL; /* Resolved later. */
4867         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4868         instr->jmp.a.n_bits = fa->n_bits;
4869         instr->jmp.a.offset = fa->offset / 8;
4870         instr->jmp.b_val = (uint32_t)b_val;
4871         return 0;
4872 }
4873
4874 static int
4875 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
4876                        struct action *action,
4877                        char **tokens,
4878                        int n_tokens,
4879                        struct instruction *instr,
4880                        struct instruction_data *data)
4881 {
4882         char *a = tokens[2], *b = tokens[3];
4883         struct field *fa, *fb;
4884         uint32_t a_struct_id, b_struct_id, b_val;
4885
4886         CHECK(n_tokens == 4, EINVAL);
4887
4888         strcpy(data->jmp_label, tokens[1]);
4889
4890         fa = struct_field_parse(p, action, a, &a_struct_id);
4891         CHECK(fa, EINVAL);
4892
4893         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
4894         fb = struct_field_parse(p, action, b, &b_struct_id);
4895         if (fb) {
4896                 instr->type = INSTR_JMP_LT;
4897                 if (a[0] == 'h' && b[0] == 'm')
4898                         instr->type = INSTR_JMP_LT_HM;
4899                 if (a[0] == 'm' && b[0] == 'h')
4900                         instr->type = INSTR_JMP_LT_MH;
4901                 if (a[0] == 'h' && b[0] == 'h')
4902                         instr->type = INSTR_JMP_LT_HH;
4903                 instr->jmp.ip = NULL; /* Resolved later. */
4904
4905                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4906                 instr->jmp.a.n_bits = fa->n_bits;
4907                 instr->jmp.a.offset = fa->offset / 8;
4908                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4909                 instr->jmp.b.n_bits = fb->n_bits;
4910                 instr->jmp.b.offset = fb->offset / 8;
4911                 return 0;
4912         }
4913
4914         /* JMP_LT_MI, JMP_LT_HI. */
4915         b_val = strtoul(b, &b, 0);
4916         CHECK(!b[0], EINVAL);
4917
4918         instr->type = INSTR_JMP_LT_MI;
4919         if (a[0] == 'h')
4920                 instr->type = INSTR_JMP_LT_HI;
4921         instr->jmp.ip = NULL; /* Resolved later. */
4922
4923         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4924         instr->jmp.a.n_bits = fa->n_bits;
4925         instr->jmp.a.offset = fa->offset / 8;
4926         instr->jmp.b_val = (uint32_t)b_val;
4927         return 0;
4928 }
4929
4930 static int
4931 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
4932                        struct action *action,
4933                        char **tokens,
4934                        int n_tokens,
4935                        struct instruction *instr,
4936                        struct instruction_data *data)
4937 {
4938         char *a = tokens[2], *b = tokens[3];
4939         struct field *fa, *fb;
4940         uint32_t a_struct_id, b_struct_id, b_val;
4941
4942         CHECK(n_tokens == 4, EINVAL);
4943
4944         strcpy(data->jmp_label, tokens[1]);
4945
4946         fa = struct_field_parse(p, action, a, &a_struct_id);
4947         CHECK(fa, EINVAL);
4948
4949         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
4950         fb = struct_field_parse(p, action, b, &b_struct_id);
4951         if (fb) {
4952                 instr->type = INSTR_JMP_GT;
4953                 if (a[0] == 'h' && b[0] == 'm')
4954                         instr->type = INSTR_JMP_GT_HM;
4955                 if (a[0] == 'm' && b[0] == 'h')
4956                         instr->type = INSTR_JMP_GT_MH;
4957                 if (a[0] == 'h' && b[0] == 'h')
4958                         instr->type = INSTR_JMP_GT_HH;
4959                 instr->jmp.ip = NULL; /* Resolved later. */
4960
4961                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4962                 instr->jmp.a.n_bits = fa->n_bits;
4963                 instr->jmp.a.offset = fa->offset / 8;
4964                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4965                 instr->jmp.b.n_bits = fb->n_bits;
4966                 instr->jmp.b.offset = fb->offset / 8;
4967                 return 0;
4968         }
4969
4970         /* JMP_GT_MI, JMP_GT_HI. */
4971         b_val = strtoul(b, &b, 0);
4972         CHECK(!b[0], EINVAL);
4973
4974         instr->type = INSTR_JMP_GT_MI;
4975         if (a[0] == 'h')
4976                 instr->type = INSTR_JMP_GT_HI;
4977         instr->jmp.ip = NULL; /* Resolved later. */
4978
4979         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4980         instr->jmp.a.n_bits = fa->n_bits;
4981         instr->jmp.a.offset = fa->offset / 8;
4982         instr->jmp.b_val = (uint32_t)b_val;
4983         return 0;
4984 }
4985
4986 static inline void
4987 instr_jmp_exec(struct rte_swx_pipeline *p)
4988 {
4989         struct thread *t = &p->threads[p->thread_id];
4990         struct instruction *ip = t->ip;
4991
4992         TRACE("[Thread %2u] jmp\n", p->thread_id);
4993
4994         thread_ip_set(t, ip->jmp.ip);
4995 }
4996
4997 static inline void
4998 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
4999 {
5000         struct thread *t = &p->threads[p->thread_id];
5001         struct instruction *ip = t->ip;
5002         uint32_t header_id = ip->jmp.header_id;
5003
5004         TRACE("[Thread %2u] jmpv\n", p->thread_id);
5005
5006         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5007 }
5008
5009 static inline void
5010 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5011 {
5012         struct thread *t = &p->threads[p->thread_id];
5013         struct instruction *ip = t->ip;
5014         uint32_t header_id = ip->jmp.header_id;
5015
5016         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5017
5018         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5019 }
5020
5021 static inline void
5022 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5023 {
5024         struct thread *t = &p->threads[p->thread_id];
5025         struct instruction *ip = t->ip;
5026         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5027
5028         TRACE("[Thread %2u] jmph\n", p->thread_id);
5029
5030         t->ip = ip_next[t->hit];
5031 }
5032
5033 static inline void
5034 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5035 {
5036         struct thread *t = &p->threads[p->thread_id];
5037         struct instruction *ip = t->ip;
5038         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5039
5040         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5041
5042         t->ip = ip_next[t->hit];
5043 }
5044
5045 static inline void
5046 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5047 {
5048         struct thread *t = &p->threads[p->thread_id];
5049         struct instruction *ip = t->ip;
5050
5051         TRACE("[Thread %2u] jmpa\n", p->thread_id);
5052
5053         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5054 }
5055
5056 static inline void
5057 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5058 {
5059         struct thread *t = &p->threads[p->thread_id];
5060         struct instruction *ip = t->ip;
5061
5062         TRACE("[Thread %2u] jmpna\n", p->thread_id);
5063
5064         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5065 }
5066
5067 static inline void
5068 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5069 {
5070         struct thread *t = &p->threads[p->thread_id];
5071         struct instruction *ip = t->ip;
5072
5073         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5074
5075         JMP_CMP(t, ip, ==);
5076 }
5077
5078 static inline void
5079 instr_jmp_eq_s_exec(struct rte_swx_pipeline *p)
5080 {
5081         struct thread *t = &p->threads[p->thread_id];
5082         struct instruction *ip = t->ip;
5083
5084         TRACE("[Thread %2u] jmpeq (s)\n", p->thread_id);
5085
5086         JMP_CMP_S(t, ip, ==);
5087 }
5088
5089 static inline void
5090 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5091 {
5092         struct thread *t = &p->threads[p->thread_id];
5093         struct instruction *ip = t->ip;
5094
5095         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5096
5097         JMP_CMP_I(t, ip, ==);
5098 }
5099
5100 static inline void
5101 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5102 {
5103         struct thread *t = &p->threads[p->thread_id];
5104         struct instruction *ip = t->ip;
5105
5106         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5107
5108         JMP_CMP(t, ip, !=);
5109 }
5110
5111 static inline void
5112 instr_jmp_neq_s_exec(struct rte_swx_pipeline *p)
5113 {
5114         struct thread *t = &p->threads[p->thread_id];
5115         struct instruction *ip = t->ip;
5116
5117         TRACE("[Thread %2u] jmpneq (s)\n", p->thread_id);
5118
5119         JMP_CMP_S(t, ip, !=);
5120 }
5121
5122 static inline void
5123 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5124 {
5125         struct thread *t = &p->threads[p->thread_id];
5126         struct instruction *ip = t->ip;
5127
5128         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5129
5130         JMP_CMP_I(t, ip, !=);
5131 }
5132
5133 static inline void
5134 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5135 {
5136         struct thread *t = &p->threads[p->thread_id];
5137         struct instruction *ip = t->ip;
5138
5139         TRACE("[Thread %2u] jmplt\n", p->thread_id);
5140
5141         JMP_CMP(t, ip, <);
5142 }
5143
5144 static inline void
5145 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5146 {
5147         struct thread *t = &p->threads[p->thread_id];
5148         struct instruction *ip = t->ip;
5149
5150         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5151
5152         JMP_CMP_MH(t, ip, <);
5153 }
5154
5155 static inline void
5156 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5157 {
5158         struct thread *t = &p->threads[p->thread_id];
5159         struct instruction *ip = t->ip;
5160
5161         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5162
5163         JMP_CMP_HM(t, ip, <);
5164 }
5165
5166 static inline void
5167 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5168 {
5169         struct thread *t = &p->threads[p->thread_id];
5170         struct instruction *ip = t->ip;
5171
5172         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5173
5174         JMP_CMP_HH(t, ip, <);
5175 }
5176
5177 static inline void
5178 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5179 {
5180         struct thread *t = &p->threads[p->thread_id];
5181         struct instruction *ip = t->ip;
5182
5183         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5184
5185         JMP_CMP_MI(t, ip, <);
5186 }
5187
5188 static inline void
5189 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5190 {
5191         struct thread *t = &p->threads[p->thread_id];
5192         struct instruction *ip = t->ip;
5193
5194         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5195
5196         JMP_CMP_HI(t, ip, <);
5197 }
5198
5199 static inline void
5200 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5201 {
5202         struct thread *t = &p->threads[p->thread_id];
5203         struct instruction *ip = t->ip;
5204
5205         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5206
5207         JMP_CMP(t, ip, >);
5208 }
5209
5210 static inline void
5211 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5212 {
5213         struct thread *t = &p->threads[p->thread_id];
5214         struct instruction *ip = t->ip;
5215
5216         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5217
5218         JMP_CMP_MH(t, ip, >);
5219 }
5220
5221 static inline void
5222 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5223 {
5224         struct thread *t = &p->threads[p->thread_id];
5225         struct instruction *ip = t->ip;
5226
5227         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5228
5229         JMP_CMP_HM(t, ip, >);
5230 }
5231
5232 static inline void
5233 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5234 {
5235         struct thread *t = &p->threads[p->thread_id];
5236         struct instruction *ip = t->ip;
5237
5238         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5239
5240         JMP_CMP_HH(t, ip, >);
5241 }
5242
5243 static inline void
5244 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5245 {
5246         struct thread *t = &p->threads[p->thread_id];
5247         struct instruction *ip = t->ip;
5248
5249         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5250
5251         JMP_CMP_MI(t, ip, >);
5252 }
5253
5254 static inline void
5255 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5256 {
5257         struct thread *t = &p->threads[p->thread_id];
5258         struct instruction *ip = t->ip;
5259
5260         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5261
5262         JMP_CMP_HI(t, ip, >);
5263 }
5264
5265 /*
5266  * return.
5267  */
5268 static int
5269 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5270                        struct action *action,
5271                        char **tokens __rte_unused,
5272                        int n_tokens,
5273                        struct instruction *instr,
5274                        struct instruction_data *data __rte_unused)
5275 {
5276         CHECK(action, EINVAL);
5277         CHECK(n_tokens == 1, EINVAL);
5278
5279         instr->type = INSTR_RETURN;
5280         return 0;
5281 }
5282
5283 static inline void
5284 instr_return_exec(struct rte_swx_pipeline *p)
5285 {
5286         struct thread *t = &p->threads[p->thread_id];
5287
5288         TRACE("[Thread %2u] return\n", p->thread_id);
5289
5290         t->ip = t->ret;
5291 }
5292
5293 static int
5294 instr_translate(struct rte_swx_pipeline *p,
5295                 struct action *action,
5296                 char *string,
5297                 struct instruction *instr,
5298                 struct instruction_data *data)
5299 {
5300         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5301         int n_tokens = 0, tpos = 0;
5302
5303         /* Parse the instruction string into tokens. */
5304         for ( ; ; ) {
5305                 char *token;
5306
5307                 token = strtok_r(string, " \t\v", &string);
5308                 if (!token)
5309                         break;
5310
5311                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5312                 CHECK_NAME(token, EINVAL);
5313
5314                 tokens[n_tokens] = token;
5315                 n_tokens++;
5316         }
5317
5318         CHECK(n_tokens, EINVAL);
5319
5320         /* Handle the optional instruction label. */
5321         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5322                 strcpy(data->label, tokens[0]);
5323
5324                 tpos += 2;
5325                 CHECK(n_tokens - tpos, EINVAL);
5326         }
5327
5328         /* Identify the instruction type. */
5329         if (!strcmp(tokens[tpos], "rx"))
5330                 return instr_rx_translate(p,
5331                                           action,
5332                                           &tokens[tpos],
5333                                           n_tokens - tpos,
5334                                           instr,
5335                                           data);
5336
5337         if (!strcmp(tokens[tpos], "tx"))
5338                 return instr_tx_translate(p,
5339                                           action,
5340                                           &tokens[tpos],
5341                                           n_tokens - tpos,
5342                                           instr,
5343                                           data);
5344
5345         if (!strcmp(tokens[tpos], "extract"))
5346                 return instr_hdr_extract_translate(p,
5347                                                    action,
5348                                                    &tokens[tpos],
5349                                                    n_tokens - tpos,
5350                                                    instr,
5351                                                    data);
5352
5353         if (!strcmp(tokens[tpos], "emit"))
5354                 return instr_hdr_emit_translate(p,
5355                                                 action,
5356                                                 &tokens[tpos],
5357                                                 n_tokens - tpos,
5358                                                 instr,
5359                                                 data);
5360
5361         if (!strcmp(tokens[tpos], "validate"))
5362                 return instr_hdr_validate_translate(p,
5363                                                     action,
5364                                                     &tokens[tpos],
5365                                                     n_tokens - tpos,
5366                                                     instr,
5367                                                     data);
5368
5369         if (!strcmp(tokens[tpos], "invalidate"))
5370                 return instr_hdr_invalidate_translate(p,
5371                                                       action,
5372                                                       &tokens[tpos],
5373                                                       n_tokens - tpos,
5374                                                       instr,
5375                                                       data);
5376
5377         if (!strcmp(tokens[tpos], "mov"))
5378                 return instr_mov_translate(p,
5379                                            action,
5380                                            &tokens[tpos],
5381                                            n_tokens - tpos,
5382                                            instr,
5383                                            data);
5384
5385         if (!strcmp(tokens[tpos], "dma"))
5386                 return instr_dma_translate(p,
5387                                            action,
5388                                            &tokens[tpos],
5389                                            n_tokens - tpos,
5390                                            instr,
5391                                            data);
5392
5393         if (!strcmp(tokens[tpos], "add"))
5394                 return instr_alu_add_translate(p,
5395                                                action,
5396                                                &tokens[tpos],
5397                                                n_tokens - tpos,
5398                                                instr,
5399                                                data);
5400
5401         if (!strcmp(tokens[tpos], "sub"))
5402                 return instr_alu_sub_translate(p,
5403                                                action,
5404                                                &tokens[tpos],
5405                                                n_tokens - tpos,
5406                                                instr,
5407                                                data);
5408
5409         if (!strcmp(tokens[tpos], "ckadd"))
5410                 return instr_alu_ckadd_translate(p,
5411                                                  action,
5412                                                  &tokens[tpos],
5413                                                  n_tokens - tpos,
5414                                                  instr,
5415                                                  data);
5416
5417         if (!strcmp(tokens[tpos], "cksub"))
5418                 return instr_alu_cksub_translate(p,
5419                                                  action,
5420                                                  &tokens[tpos],
5421                                                  n_tokens - tpos,
5422                                                  instr,
5423                                                  data);
5424
5425         if (!strcmp(tokens[tpos], "and"))
5426                 return instr_alu_and_translate(p,
5427                                                action,
5428                                                &tokens[tpos],
5429                                                n_tokens - tpos,
5430                                                instr,
5431                                                data);
5432
5433         if (!strcmp(tokens[tpos], "or"))
5434                 return instr_alu_or_translate(p,
5435                                               action,
5436                                               &tokens[tpos],
5437                                               n_tokens - tpos,
5438                                               instr,
5439                                               data);
5440
5441         if (!strcmp(tokens[tpos], "xor"))
5442                 return instr_alu_xor_translate(p,
5443                                                action,
5444                                                &tokens[tpos],
5445                                                n_tokens - tpos,
5446                                                instr,
5447                                                data);
5448
5449         if (!strcmp(tokens[tpos], "shl"))
5450                 return instr_alu_shl_translate(p,
5451                                                action,
5452                                                &tokens[tpos],
5453                                                n_tokens - tpos,
5454                                                instr,
5455                                                data);
5456
5457         if (!strcmp(tokens[tpos], "shr"))
5458                 return instr_alu_shr_translate(p,
5459                                                action,
5460                                                &tokens[tpos],
5461                                                n_tokens - tpos,
5462                                                instr,
5463                                                data);
5464
5465         if (!strcmp(tokens[tpos], "table"))
5466                 return instr_table_translate(p,
5467                                              action,
5468                                              &tokens[tpos],
5469                                              n_tokens - tpos,
5470                                              instr,
5471                                              data);
5472
5473         if (!strcmp(tokens[tpos], "extern"))
5474                 return instr_extern_translate(p,
5475                                               action,
5476                                               &tokens[tpos],
5477                                               n_tokens - tpos,
5478                                               instr,
5479                                               data);
5480
5481         if (!strcmp(tokens[tpos], "jmp"))
5482                 return instr_jmp_translate(p,
5483                                            action,
5484                                            &tokens[tpos],
5485                                            n_tokens - tpos,
5486                                            instr,
5487                                            data);
5488
5489         if (!strcmp(tokens[tpos], "jmpv"))
5490                 return instr_jmp_valid_translate(p,
5491                                                  action,
5492                                                  &tokens[tpos],
5493                                                  n_tokens - tpos,
5494                                                  instr,
5495                                                  data);
5496
5497         if (!strcmp(tokens[tpos], "jmpnv"))
5498                 return instr_jmp_invalid_translate(p,
5499                                                    action,
5500                                                    &tokens[tpos],
5501                                                    n_tokens - tpos,
5502                                                    instr,
5503                                                    data);
5504
5505         if (!strcmp(tokens[tpos], "jmph"))
5506                 return instr_jmp_hit_translate(p,
5507                                                action,
5508                                                &tokens[tpos],
5509                                                n_tokens - tpos,
5510                                                instr,
5511                                                data);
5512
5513         if (!strcmp(tokens[tpos], "jmpnh"))
5514                 return instr_jmp_miss_translate(p,
5515                                                 action,
5516                                                 &tokens[tpos],
5517                                                 n_tokens - tpos,
5518                                                 instr,
5519                                                 data);
5520
5521         if (!strcmp(tokens[tpos], "jmpa"))
5522                 return instr_jmp_action_hit_translate(p,
5523                                                       action,
5524                                                       &tokens[tpos],
5525                                                       n_tokens - tpos,
5526                                                       instr,
5527                                                       data);
5528
5529         if (!strcmp(tokens[tpos], "jmpna"))
5530                 return instr_jmp_action_miss_translate(p,
5531                                                        action,
5532                                                        &tokens[tpos],
5533                                                        n_tokens - tpos,
5534                                                        instr,
5535                                                        data);
5536
5537         if (!strcmp(tokens[tpos], "jmpeq"))
5538                 return instr_jmp_eq_translate(p,
5539                                               action,
5540                                               &tokens[tpos],
5541                                               n_tokens - tpos,
5542                                               instr,
5543                                               data);
5544
5545         if (!strcmp(tokens[tpos], "jmpneq"))
5546                 return instr_jmp_neq_translate(p,
5547                                                action,
5548                                                &tokens[tpos],
5549                                                n_tokens - tpos,
5550                                                instr,
5551                                                data);
5552
5553         if (!strcmp(tokens[tpos], "jmplt"))
5554                 return instr_jmp_lt_translate(p,
5555                                               action,
5556                                               &tokens[tpos],
5557                                               n_tokens - tpos,
5558                                               instr,
5559                                               data);
5560
5561         if (!strcmp(tokens[tpos], "jmpgt"))
5562                 return instr_jmp_gt_translate(p,
5563                                               action,
5564                                               &tokens[tpos],
5565                                               n_tokens - tpos,
5566                                               instr,
5567                                               data);
5568
5569         if (!strcmp(tokens[tpos], "return"))
5570                 return instr_return_translate(p,
5571                                               action,
5572                                               &tokens[tpos],
5573                                               n_tokens - tpos,
5574                                               instr,
5575                                               data);
5576
5577         CHECK(0, EINVAL);
5578 }
5579
5580 static struct instruction_data *
5581 label_find(struct instruction_data *data, uint32_t n, const char *label)
5582 {
5583         uint32_t i;
5584
5585         for (i = 0; i < n; i++)
5586                 if (!strcmp(label, data[i].label))
5587                         return &data[i];
5588
5589         return NULL;
5590 }
5591
5592 static uint32_t
5593 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5594 {
5595         uint32_t count = 0, i;
5596
5597         if (!label[0])
5598                 return 0;
5599
5600         for (i = 0; i < n; i++)
5601                 if (!strcmp(label, data[i].jmp_label))
5602                         count++;
5603
5604         return count;
5605 }
5606
5607 static int
5608 instr_label_check(struct instruction_data *instruction_data,
5609                   uint32_t n_instructions)
5610 {
5611         uint32_t i;
5612
5613         /* Check that all instruction labels are unique. */
5614         for (i = 0; i < n_instructions; i++) {
5615                 struct instruction_data *data = &instruction_data[i];
5616                 char *label = data->label;
5617                 uint32_t j;
5618
5619                 if (!label[0])
5620                         continue;
5621
5622                 for (j = i + 1; j < n_instructions; j++)
5623                         CHECK(strcmp(label, data[j].label), EINVAL);
5624         }
5625
5626         /* Get users for each instruction label. */
5627         for (i = 0; i < n_instructions; i++) {
5628                 struct instruction_data *data = &instruction_data[i];
5629                 char *label = data->label;
5630
5631                 data->n_users = label_is_used(instruction_data,
5632                                               n_instructions,
5633                                               label);
5634         }
5635
5636         return 0;
5637 }
5638
5639 static int
5640 instr_jmp_resolve(struct instruction *instructions,
5641                   struct instruction_data *instruction_data,
5642                   uint32_t n_instructions)
5643 {
5644         uint32_t i;
5645
5646         for (i = 0; i < n_instructions; i++) {
5647                 struct instruction *instr = &instructions[i];
5648                 struct instruction_data *data = &instruction_data[i];
5649                 struct instruction_data *found;
5650
5651                 if (!instruction_is_jmp(instr))
5652                         continue;
5653
5654                 found = label_find(instruction_data,
5655                                    n_instructions,
5656                                    data->jmp_label);
5657                 CHECK(found, EINVAL);
5658
5659                 instr->jmp.ip = &instructions[found - instruction_data];
5660         }
5661
5662         return 0;
5663 }
5664
5665 static int
5666 instr_verify(struct rte_swx_pipeline *p __rte_unused,
5667              struct action *a,
5668              struct instruction *instr,
5669              struct instruction_data *data __rte_unused,
5670              uint32_t n_instructions)
5671 {
5672         if (!a) {
5673                 enum instruction_type type;
5674                 uint32_t i;
5675
5676                 /* Check that the first instruction is rx. */
5677                 CHECK(instr[0].type == INSTR_RX, EINVAL);
5678
5679                 /* Check that there is at least one tx instruction. */
5680                 for (i = 0; i < n_instructions; i++) {
5681                         type = instr[i].type;
5682
5683                         if (type == INSTR_TX)
5684                                 break;
5685                 }
5686                 CHECK(i < n_instructions, EINVAL);
5687
5688                 /* Check that the last instruction is either tx or unconditional
5689                  * jump.
5690                  */
5691                 type = instr[n_instructions - 1].type;
5692                 CHECK((type == INSTR_TX) || (type == INSTR_JMP), EINVAL);
5693         }
5694
5695         if (a) {
5696                 enum instruction_type type;
5697                 uint32_t i;
5698
5699                 /* Check that there is at least one return or tx instruction. */
5700                 for (i = 0; i < n_instructions; i++) {
5701                         type = instr[i].type;
5702
5703                         if ((type == INSTR_RETURN) || (type == INSTR_TX))
5704                                 break;
5705                 }
5706                 CHECK(i < n_instructions, EINVAL);
5707         }
5708
5709         return 0;
5710 }
5711
5712 static int
5713 instr_pattern_extract_many_detect(struct instruction *instr,
5714                                   struct instruction_data *data,
5715                                   uint32_t n_instr,
5716                                   uint32_t *n_pattern_instr)
5717 {
5718         uint32_t i;
5719
5720         for (i = 0; i < n_instr; i++) {
5721                 if (data[i].invalid)
5722                         break;
5723
5724                 if (instr[i].type != INSTR_HDR_EXTRACT)
5725                         break;
5726
5727                 if (i == RTE_DIM(instr->io.hdr.header_id))
5728                         break;
5729
5730                 if (i && data[i].n_users)
5731                         break;
5732         }
5733
5734         if (i < 2)
5735                 return 0;
5736
5737         *n_pattern_instr = i;
5738         return 1;
5739 }
5740
5741 static void
5742 instr_pattern_extract_many_optimize(struct instruction *instr,
5743                                     struct instruction_data *data,
5744                                     uint32_t n_instr)
5745 {
5746         uint32_t i;
5747
5748         for (i = 1; i < n_instr; i++) {
5749                 instr[0].type++;
5750                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
5751                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
5752                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
5753
5754                 data[i].invalid = 1;
5755         }
5756 }
5757
5758 static int
5759 instr_pattern_emit_many_tx_detect(struct instruction *instr,
5760                                   struct instruction_data *data,
5761                                   uint32_t n_instr,
5762                                   uint32_t *n_pattern_instr)
5763 {
5764         uint32_t i;
5765
5766         for (i = 0; i < n_instr; i++) {
5767                 if (data[i].invalid)
5768                         break;
5769
5770                 if (instr[i].type != INSTR_HDR_EMIT)
5771                         break;
5772
5773                 if (i == RTE_DIM(instr->io.hdr.header_id))
5774                         break;
5775
5776                 if (i && data[i].n_users)
5777                         break;
5778         }
5779
5780         if (!i)
5781                 return 0;
5782
5783         if (instr[i].type != INSTR_TX)
5784                 return 0;
5785
5786         i++;
5787
5788         *n_pattern_instr = i;
5789         return 1;
5790 }
5791
5792 static void
5793 instr_pattern_emit_many_tx_optimize(struct instruction *instr,
5794                                     struct instruction_data *data,
5795                                     uint32_t n_instr)
5796 {
5797         uint32_t i;
5798
5799         /* Any emit instruction in addition to the first one. */
5800         for (i = 1; i < n_instr - 1; i++) {
5801                 instr[0].type++;
5802                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
5803                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
5804                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
5805
5806                 data[i].invalid = 1;
5807         }
5808
5809         /* The TX instruction is the last one in the pattern. */
5810         instr[0].type++;
5811         instr[0].io.io.offset = instr[i].io.io.offset;
5812         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
5813         data[i].invalid = 1;
5814 }
5815
5816 static int
5817 instr_pattern_dma_many_detect(struct instruction *instr,
5818                               struct instruction_data *data,
5819                               uint32_t n_instr,
5820                               uint32_t *n_pattern_instr)
5821 {
5822         uint32_t i;
5823
5824         for (i = 0; i < n_instr; i++) {
5825                 if (data[i].invalid)
5826                         break;
5827
5828                 if (instr[i].type != INSTR_DMA_HT)
5829                         break;
5830
5831                 if (i == RTE_DIM(instr->dma.dst.header_id))
5832                         break;
5833
5834                 if (i && data[i].n_users)
5835                         break;
5836         }
5837
5838         if (i < 2)
5839                 return 0;
5840
5841         *n_pattern_instr = i;
5842         return 1;
5843 }
5844
5845 static void
5846 instr_pattern_dma_many_optimize(struct instruction *instr,
5847                                 struct instruction_data *data,
5848                                 uint32_t n_instr)
5849 {
5850         uint32_t i;
5851
5852         for (i = 1; i < n_instr; i++) {
5853                 instr[0].type++;
5854                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
5855                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
5856                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
5857                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
5858
5859                 data[i].invalid = 1;
5860         }
5861 }
5862
5863 static uint32_t
5864 instr_optimize(struct instruction *instructions,
5865                struct instruction_data *instruction_data,
5866                uint32_t n_instructions)
5867 {
5868         uint32_t i, pos = 0;
5869
5870         for (i = 0; i < n_instructions; ) {
5871                 struct instruction *instr = &instructions[i];
5872                 struct instruction_data *data = &instruction_data[i];
5873                 uint32_t n_instr = 0;
5874                 int detected;
5875
5876                 /* Extract many. */
5877                 detected = instr_pattern_extract_many_detect(instr,
5878                                                              data,
5879                                                              n_instructions - i,
5880                                                              &n_instr);
5881                 if (detected) {
5882                         instr_pattern_extract_many_optimize(instr,
5883                                                             data,
5884                                                             n_instr);
5885                         i += n_instr;
5886                         continue;
5887                 }
5888
5889                 /* Emit many + TX. */
5890                 detected = instr_pattern_emit_many_tx_detect(instr,
5891                                                              data,
5892                                                              n_instructions - i,
5893                                                              &n_instr);
5894                 if (detected) {
5895                         instr_pattern_emit_many_tx_optimize(instr,
5896                                                             data,
5897                                                             n_instr);
5898                         i += n_instr;
5899                         continue;
5900                 }
5901
5902                 /* DMA many. */
5903                 detected = instr_pattern_dma_many_detect(instr,
5904                                                          data,
5905                                                          n_instructions - i,
5906                                                          &n_instr);
5907                 if (detected) {
5908                         instr_pattern_dma_many_optimize(instr, data, n_instr);
5909                         i += n_instr;
5910                         continue;
5911                 }
5912
5913                 /* No pattern starting at the current instruction. */
5914                 i++;
5915         }
5916
5917         /* Eliminate the invalid instructions that have been optimized out. */
5918         for (i = 0; i < n_instructions; i++) {
5919                 struct instruction *instr = &instructions[i];
5920                 struct instruction_data *data = &instruction_data[i];
5921
5922                 if (data->invalid)
5923                         continue;
5924
5925                 if (i != pos) {
5926                         memcpy(&instructions[pos], instr, sizeof(*instr));
5927                         memcpy(&instruction_data[pos], data, sizeof(*data));
5928                 }
5929
5930                 pos++;
5931         }
5932
5933         return pos;
5934 }
5935
5936 static int
5937 instruction_config(struct rte_swx_pipeline *p,
5938                    struct action *a,
5939                    const char **instructions,
5940                    uint32_t n_instructions)
5941 {
5942         struct instruction *instr = NULL;
5943         struct instruction_data *data = NULL;
5944         int err = 0;
5945         uint32_t i;
5946
5947         CHECK(n_instructions, EINVAL);
5948         CHECK(instructions, EINVAL);
5949         for (i = 0; i < n_instructions; i++)
5950                 CHECK_INSTRUCTION(instructions[i], EINVAL);
5951
5952         /* Memory allocation. */
5953         instr = calloc(n_instructions, sizeof(struct instruction));
5954         if (!instr) {
5955                 err = ENOMEM;
5956                 goto error;
5957         }
5958
5959         data = calloc(n_instructions, sizeof(struct instruction_data));
5960         if (!data) {
5961                 err = ENOMEM;
5962                 goto error;
5963         }
5964
5965         for (i = 0; i < n_instructions; i++) {
5966                 char *string = strdup(instructions[i]);
5967                 if (!string) {
5968                         err = ENOMEM;
5969                         goto error;
5970                 }
5971
5972                 err = instr_translate(p, a, string, &instr[i], &data[i]);
5973                 if (err) {
5974                         free(string);
5975                         goto error;
5976                 }
5977
5978                 free(string);
5979         }
5980
5981         err = instr_label_check(data, n_instructions);
5982         if (err)
5983                 goto error;
5984
5985         err = instr_verify(p, a, instr, data, n_instructions);
5986         if (err)
5987                 goto error;
5988
5989         n_instructions = instr_optimize(instr, data, n_instructions);
5990
5991         err = instr_jmp_resolve(instr, data, n_instructions);
5992         if (err)
5993                 goto error;
5994
5995         if (a) {
5996                 a->instructions = instr;
5997                 a->n_instructions = n_instructions;
5998         } else {
5999                 p->instructions = instr;
6000                 p->n_instructions = n_instructions;
6001         }
6002
6003         free(data);
6004         return 0;
6005
6006 error:
6007         free(data);
6008         free(instr);
6009         return err;
6010 }
6011
6012 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
6013
6014 static instr_exec_t instruction_table[] = {
6015         [INSTR_RX] = instr_rx_exec,
6016         [INSTR_TX] = instr_tx_exec,
6017
6018         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6019         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6020         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6021         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6022         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6023         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6024         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6025         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6026
6027         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6028         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6029         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6030         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6031         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6032         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6033         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6034         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6035         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6036
6037         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6038         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6039
6040         [INSTR_MOV] = instr_mov_exec,
6041         [INSTR_MOV_S] = instr_mov_s_exec,
6042         [INSTR_MOV_I] = instr_mov_i_exec,
6043
6044         [INSTR_DMA_HT] = instr_dma_ht_exec,
6045         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6046         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6047         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6048         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6049         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6050         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6051         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6052
6053         [INSTR_ALU_ADD] = instr_alu_add_exec,
6054         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6055         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6056         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6057         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6058         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6059
6060         [INSTR_ALU_SUB] = instr_alu_sub_exec,
6061         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6062         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6063         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6064         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6065         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6066
6067         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6068         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6069         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6070         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6071
6072         [INSTR_ALU_AND] = instr_alu_and_exec,
6073         [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
6074         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6075
6076         [INSTR_ALU_OR] = instr_alu_or_exec,
6077         [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
6078         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6079
6080         [INSTR_ALU_XOR] = instr_alu_xor_exec,
6081         [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec,
6082         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6083
6084         [INSTR_ALU_SHL] = instr_alu_shl_exec,
6085         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6086         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6087         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6088         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6089         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6090
6091         [INSTR_ALU_SHR] = instr_alu_shr_exec,
6092         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6093         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6094         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6095         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6096         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6097
6098         [INSTR_TABLE] = instr_table_exec,
6099         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6100         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6101
6102         [INSTR_JMP] = instr_jmp_exec,
6103         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6104         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6105         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6106         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6107         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6108         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6109
6110         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6111         [INSTR_JMP_EQ_S] = instr_jmp_eq_s_exec,
6112         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6113
6114         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6115         [INSTR_JMP_NEQ_S] = instr_jmp_neq_s_exec,
6116         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6117
6118         [INSTR_JMP_LT] = instr_jmp_lt_exec,
6119         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6120         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6121         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6122         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6123         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6124
6125         [INSTR_JMP_GT] = instr_jmp_gt_exec,
6126         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6127         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6128         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6129         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6130         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6131
6132         [INSTR_RETURN] = instr_return_exec,
6133 };
6134
6135 static inline void
6136 instr_exec(struct rte_swx_pipeline *p)
6137 {
6138         struct thread *t = &p->threads[p->thread_id];
6139         struct instruction *ip = t->ip;
6140         instr_exec_t instr = instruction_table[ip->type];
6141
6142         instr(p);
6143 }
6144
6145 /*
6146  * Action.
6147  */
6148 static struct action *
6149 action_find(struct rte_swx_pipeline *p, const char *name)
6150 {
6151         struct action *elem;
6152
6153         if (!name)
6154                 return NULL;
6155
6156         TAILQ_FOREACH(elem, &p->actions, node)
6157                 if (strcmp(elem->name, name) == 0)
6158                         return elem;
6159
6160         return NULL;
6161 }
6162
6163 static struct action *
6164 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6165 {
6166         struct action *action = NULL;
6167
6168         TAILQ_FOREACH(action, &p->actions, node)
6169                 if (action->id == id)
6170                         return action;
6171
6172         return NULL;
6173 }
6174
6175 static struct field *
6176 action_field_find(struct action *a, const char *name)
6177 {
6178         return a->st ? struct_type_field_find(a->st, name) : NULL;
6179 }
6180
6181 static struct field *
6182 action_field_parse(struct action *action, const char *name)
6183 {
6184         if (name[0] != 't' || name[1] != '.')
6185                 return NULL;
6186
6187         return action_field_find(action, &name[2]);
6188 }
6189
6190 int
6191 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6192                                const char *name,
6193                                const char *args_struct_type_name,
6194                                const char **instructions,
6195                                uint32_t n_instructions)
6196 {
6197         struct struct_type *args_struct_type;
6198         struct action *a;
6199         int err;
6200
6201         CHECK(p, EINVAL);
6202
6203         CHECK_NAME(name, EINVAL);
6204         CHECK(!action_find(p, name), EEXIST);
6205
6206         if (args_struct_type_name) {
6207                 CHECK_NAME(args_struct_type_name, EINVAL);
6208                 args_struct_type = struct_type_find(p, args_struct_type_name);
6209                 CHECK(args_struct_type, EINVAL);
6210         } else {
6211                 args_struct_type = NULL;
6212         }
6213
6214         /* Node allocation. */
6215         a = calloc(1, sizeof(struct action));
6216         CHECK(a, ENOMEM);
6217
6218         /* Node initialization. */
6219         strcpy(a->name, name);
6220         a->st = args_struct_type;
6221         a->id = p->n_actions;
6222
6223         /* Instruction translation. */
6224         err = instruction_config(p, a, instructions, n_instructions);
6225         if (err) {
6226                 free(a);
6227                 return err;
6228         }
6229
6230         /* Node add to tailq. */
6231         TAILQ_INSERT_TAIL(&p->actions, a, node);
6232         p->n_actions++;
6233
6234         return 0;
6235 }
6236
6237 static int
6238 action_build(struct rte_swx_pipeline *p)
6239 {
6240         struct action *action;
6241
6242         p->action_instructions = calloc(p->n_actions,
6243                                         sizeof(struct instruction *));
6244         CHECK(p->action_instructions, ENOMEM);
6245
6246         TAILQ_FOREACH(action, &p->actions, node)
6247                 p->action_instructions[action->id] = action->instructions;
6248
6249         return 0;
6250 }
6251
6252 static void
6253 action_build_free(struct rte_swx_pipeline *p)
6254 {
6255         free(p->action_instructions);
6256         p->action_instructions = NULL;
6257 }
6258
6259 static void
6260 action_free(struct rte_swx_pipeline *p)
6261 {
6262         action_build_free(p);
6263
6264         for ( ; ; ) {
6265                 struct action *action;
6266
6267                 action = TAILQ_FIRST(&p->actions);
6268                 if (!action)
6269                         break;
6270
6271                 TAILQ_REMOVE(&p->actions, action, node);
6272                 free(action->instructions);
6273                 free(action);
6274         }
6275 }
6276
6277 /*
6278  * Table.
6279  */
6280 static struct table_type *
6281 table_type_find(struct rte_swx_pipeline *p, const char *name)
6282 {
6283         struct table_type *elem;
6284
6285         TAILQ_FOREACH(elem, &p->table_types, node)
6286                 if (strcmp(elem->name, name) == 0)
6287                         return elem;
6288
6289         return NULL;
6290 }
6291
6292 static struct table_type *
6293 table_type_resolve(struct rte_swx_pipeline *p,
6294                    const char *recommended_type_name,
6295                    enum rte_swx_table_match_type match_type)
6296 {
6297         struct table_type *elem;
6298
6299         /* Only consider the recommended type if the match type is correct. */
6300         if (recommended_type_name)
6301                 TAILQ_FOREACH(elem, &p->table_types, node)
6302                         if (!strcmp(elem->name, recommended_type_name) &&
6303                             (elem->match_type == match_type))
6304                                 return elem;
6305
6306         /* Ignore the recommended type and get the first element with this match
6307          * type.
6308          */
6309         TAILQ_FOREACH(elem, &p->table_types, node)
6310                 if (elem->match_type == match_type)
6311                         return elem;
6312
6313         return NULL;
6314 }
6315
6316 static struct table *
6317 table_find(struct rte_swx_pipeline *p, const char *name)
6318 {
6319         struct table *elem;
6320
6321         TAILQ_FOREACH(elem, &p->tables, node)
6322                 if (strcmp(elem->name, name) == 0)
6323                         return elem;
6324
6325         return NULL;
6326 }
6327
6328 static struct table *
6329 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6330 {
6331         struct table *table = NULL;
6332
6333         TAILQ_FOREACH(table, &p->tables, node)
6334                 if (table->id == id)
6335                         return table;
6336
6337         return NULL;
6338 }
6339
6340 int
6341 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
6342                                      const char *name,
6343                                      enum rte_swx_table_match_type match_type,
6344                                      struct rte_swx_table_ops *ops)
6345 {
6346         struct table_type *elem;
6347
6348         CHECK(p, EINVAL);
6349
6350         CHECK_NAME(name, EINVAL);
6351         CHECK(!table_type_find(p, name), EEXIST);
6352
6353         CHECK(ops, EINVAL);
6354         CHECK(ops->create, EINVAL);
6355         CHECK(ops->lkp, EINVAL);
6356         CHECK(ops->free, EINVAL);
6357
6358         /* Node allocation. */
6359         elem = calloc(1, sizeof(struct table_type));
6360         CHECK(elem, ENOMEM);
6361
6362         /* Node initialization. */
6363         strcpy(elem->name, name);
6364         elem->match_type = match_type;
6365         memcpy(&elem->ops, ops, sizeof(*ops));
6366
6367         /* Node add to tailq. */
6368         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
6369
6370         return 0;
6371 }
6372
6373 static enum rte_swx_table_match_type
6374 table_match_type_resolve(struct rte_swx_match_field_params *fields,
6375                          uint32_t n_fields)
6376 {
6377         uint32_t i;
6378
6379         for (i = 0; i < n_fields; i++)
6380                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
6381                         break;
6382
6383         if (i == n_fields)
6384                 return RTE_SWX_TABLE_MATCH_EXACT;
6385
6386         if ((i == n_fields - 1) &&
6387             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
6388                 return RTE_SWX_TABLE_MATCH_LPM;
6389
6390         return RTE_SWX_TABLE_MATCH_WILDCARD;
6391 }
6392
6393 int
6394 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
6395                               const char *name,
6396                               struct rte_swx_pipeline_table_params *params,
6397                               const char *recommended_table_type_name,
6398                               const char *args,
6399                               uint32_t size)
6400 {
6401         struct table_type *type;
6402         struct table *t;
6403         struct action *default_action;
6404         struct header *header = NULL;
6405         int is_header = 0;
6406         uint32_t offset_prev = 0, action_data_size_max = 0, i;
6407
6408         CHECK(p, EINVAL);
6409
6410         CHECK_NAME(name, EINVAL);
6411         CHECK(!table_find(p, name), EEXIST);
6412
6413         CHECK(params, EINVAL);
6414
6415         /* Match checks. */
6416         CHECK(!params->n_fields || params->fields, EINVAL);
6417         for (i = 0; i < params->n_fields; i++) {
6418                 struct rte_swx_match_field_params *field = &params->fields[i];
6419                 struct header *h;
6420                 struct field *hf, *mf;
6421                 uint32_t offset;
6422
6423                 CHECK_NAME(field->name, EINVAL);
6424
6425                 hf = header_field_parse(p, field->name, &h);
6426                 mf = metadata_field_parse(p, field->name);
6427                 CHECK(hf || mf, EINVAL);
6428
6429                 offset = hf ? hf->offset : mf->offset;
6430
6431                 if (i == 0) {
6432                         is_header = hf ? 1 : 0;
6433                         header = hf ? h : NULL;
6434                         offset_prev = offset;
6435
6436                         continue;
6437                 }
6438
6439                 CHECK((is_header && hf && (h->id == header->id)) ||
6440                       (!is_header && mf), EINVAL);
6441
6442                 CHECK(offset > offset_prev, EINVAL);
6443                 offset_prev = offset;
6444         }
6445
6446         /* Action checks. */
6447         CHECK(params->n_actions, EINVAL);
6448         CHECK(params->action_names, EINVAL);
6449         for (i = 0; i < params->n_actions; i++) {
6450                 const char *action_name = params->action_names[i];
6451                 struct action *a;
6452                 uint32_t action_data_size;
6453
6454                 CHECK_NAME(action_name, EINVAL);
6455
6456                 a = action_find(p, action_name);
6457                 CHECK(a, EINVAL);
6458
6459                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
6460                 if (action_data_size > action_data_size_max)
6461                         action_data_size_max = action_data_size;
6462         }
6463
6464         CHECK_NAME(params->default_action_name, EINVAL);
6465         for (i = 0; i < p->n_actions; i++)
6466                 if (!strcmp(params->action_names[i],
6467                             params->default_action_name))
6468                         break;
6469         CHECK(i < params->n_actions, EINVAL);
6470         default_action = action_find(p, params->default_action_name);
6471         CHECK((default_action->st && params->default_action_data) ||
6472               !params->default_action_data, EINVAL);
6473
6474         /* Table type checks. */
6475         if (recommended_table_type_name)
6476                 CHECK_NAME(recommended_table_type_name, EINVAL);
6477
6478         if (params->n_fields) {
6479                 enum rte_swx_table_match_type match_type;
6480
6481                 match_type = table_match_type_resolve(params->fields,
6482                                                       params->n_fields);
6483                 type = table_type_resolve(p,
6484                                           recommended_table_type_name,
6485                                           match_type);
6486                 CHECK(type, EINVAL);
6487         } else {
6488                 type = NULL;
6489         }
6490
6491         /* Memory allocation. */
6492         t = calloc(1, sizeof(struct table));
6493         CHECK(t, ENOMEM);
6494
6495         t->fields = calloc(params->n_fields, sizeof(struct match_field));
6496         if (!t->fields) {
6497                 free(t);
6498                 CHECK(0, ENOMEM);
6499         }
6500
6501         t->actions = calloc(params->n_actions, sizeof(struct action *));
6502         if (!t->actions) {
6503                 free(t->fields);
6504                 free(t);
6505                 CHECK(0, ENOMEM);
6506         }
6507
6508         if (action_data_size_max) {
6509                 t->default_action_data = calloc(1, action_data_size_max);
6510                 if (!t->default_action_data) {
6511                         free(t->actions);
6512                         free(t->fields);
6513                         free(t);
6514                         CHECK(0, ENOMEM);
6515                 }
6516         }
6517
6518         /* Node initialization. */
6519         strcpy(t->name, name);
6520         if (args && args[0])
6521                 strcpy(t->args, args);
6522         t->type = type;
6523
6524         for (i = 0; i < params->n_fields; i++) {
6525                 struct rte_swx_match_field_params *field = &params->fields[i];
6526                 struct match_field *f = &t->fields[i];
6527
6528                 f->match_type = field->match_type;
6529                 f->field = is_header ?
6530                         header_field_parse(p, field->name, NULL) :
6531                         metadata_field_parse(p, field->name);
6532         }
6533         t->n_fields = params->n_fields;
6534         t->is_header = is_header;
6535         t->header = header;
6536
6537         for (i = 0; i < params->n_actions; i++)
6538                 t->actions[i] = action_find(p, params->action_names[i]);
6539         t->default_action = default_action;
6540         if (default_action->st)
6541                 memcpy(t->default_action_data,
6542                        params->default_action_data,
6543                        default_action->st->n_bits / 8);
6544         t->n_actions = params->n_actions;
6545         t->default_action_is_const = params->default_action_is_const;
6546         t->action_data_size_max = action_data_size_max;
6547
6548         t->size = size;
6549         t->id = p->n_tables;
6550
6551         /* Node add to tailq. */
6552         TAILQ_INSERT_TAIL(&p->tables, t, node);
6553         p->n_tables++;
6554
6555         return 0;
6556 }
6557
6558 static struct rte_swx_table_params *
6559 table_params_get(struct table *table)
6560 {
6561         struct rte_swx_table_params *params;
6562         struct field *first, *last;
6563         uint8_t *key_mask;
6564         uint32_t key_size, key_offset, action_data_size, i;
6565
6566         /* Memory allocation. */
6567         params = calloc(1, sizeof(struct rte_swx_table_params));
6568         if (!params)
6569                 return NULL;
6570
6571         /* Key offset and size. */
6572         first = table->fields[0].field;
6573         last = table->fields[table->n_fields - 1].field;
6574         key_offset = first->offset / 8;
6575         key_size = (last->offset + last->n_bits - first->offset) / 8;
6576
6577         /* Memory allocation. */
6578         key_mask = calloc(1, key_size);
6579         if (!key_mask) {
6580                 free(params);
6581                 return NULL;
6582         }
6583
6584         /* Key mask. */
6585         for (i = 0; i < table->n_fields; i++) {
6586                 struct field *f = table->fields[i].field;
6587                 uint32_t start = (f->offset - first->offset) / 8;
6588                 size_t size = f->n_bits / 8;
6589
6590                 memset(&key_mask[start], 0xFF, size);
6591         }
6592
6593         /* Action data size. */
6594         action_data_size = 0;
6595         for (i = 0; i < table->n_actions; i++) {
6596                 struct action *action = table->actions[i];
6597                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
6598
6599                 if (ads > action_data_size)
6600                         action_data_size = ads;
6601         }
6602
6603         /* Fill in. */
6604         params->match_type = table->type->match_type;
6605         params->key_size = key_size;
6606         params->key_offset = key_offset;
6607         params->key_mask0 = key_mask;
6608         params->action_data_size = action_data_size;
6609         params->n_keys_max = table->size;
6610
6611         return params;
6612 }
6613
6614 static void
6615 table_params_free(struct rte_swx_table_params *params)
6616 {
6617         if (!params)
6618                 return;
6619
6620         free(params->key_mask0);
6621         free(params);
6622 }
6623
6624 static int
6625 table_state_build(struct rte_swx_pipeline *p)
6626 {
6627         struct table *table;
6628
6629         p->table_state = calloc(p->n_tables,
6630                                 sizeof(struct rte_swx_table_state));
6631         CHECK(p->table_state, ENOMEM);
6632
6633         TAILQ_FOREACH(table, &p->tables, node) {
6634                 struct rte_swx_table_state *ts = &p->table_state[table->id];
6635
6636                 if (table->type) {
6637                         struct rte_swx_table_params *params;
6638
6639                         /* ts->obj. */
6640                         params = table_params_get(table);
6641                         CHECK(params, ENOMEM);
6642
6643                         ts->obj = table->type->ops.create(params,
6644                                 NULL,
6645                                 table->args,
6646                                 p->numa_node);
6647
6648                         table_params_free(params);
6649                         CHECK(ts->obj, ENODEV);
6650                 }
6651
6652                 /* ts->default_action_data. */
6653                 if (table->action_data_size_max) {
6654                         ts->default_action_data =
6655                                 malloc(table->action_data_size_max);
6656                         CHECK(ts->default_action_data, ENOMEM);
6657
6658                         memcpy(ts->default_action_data,
6659                                table->default_action_data,
6660                                table->action_data_size_max);
6661                 }
6662
6663                 /* ts->default_action_id. */
6664                 ts->default_action_id = table->default_action->id;
6665         }
6666
6667         return 0;
6668 }
6669
6670 static void
6671 table_state_build_free(struct rte_swx_pipeline *p)
6672 {
6673         uint32_t i;
6674
6675         if (!p->table_state)
6676                 return;
6677
6678         for (i = 0; i < p->n_tables; i++) {
6679                 struct rte_swx_table_state *ts = &p->table_state[i];
6680                 struct table *table = table_find_by_id(p, i);
6681
6682                 /* ts->obj. */
6683                 if (table->type && ts->obj)
6684                         table->type->ops.free(ts->obj);
6685
6686                 /* ts->default_action_data. */
6687                 free(ts->default_action_data);
6688         }
6689
6690         free(p->table_state);
6691         p->table_state = NULL;
6692 }
6693
6694 static void
6695 table_state_free(struct rte_swx_pipeline *p)
6696 {
6697         table_state_build_free(p);
6698 }
6699
6700 static int
6701 table_stub_lkp(void *table __rte_unused,
6702                void *mailbox __rte_unused,
6703                uint8_t **key __rte_unused,
6704                uint64_t *action_id __rte_unused,
6705                uint8_t **action_data __rte_unused,
6706                int *hit)
6707 {
6708         *hit = 0;
6709         return 1; /* DONE. */
6710 }
6711
6712 static int
6713 table_build(struct rte_swx_pipeline *p)
6714 {
6715         uint32_t i;
6716
6717         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
6718                 struct thread *t = &p->threads[i];
6719                 struct table *table;
6720
6721                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
6722                 CHECK(t->tables, ENOMEM);
6723
6724                 TAILQ_FOREACH(table, &p->tables, node) {
6725                         struct table_runtime *r = &t->tables[table->id];
6726
6727                         if (table->type) {
6728                                 uint64_t size;
6729
6730                                 size = table->type->ops.mailbox_size_get();
6731
6732                                 /* r->func. */
6733                                 r->func = table->type->ops.lkp;
6734
6735                                 /* r->mailbox. */
6736                                 if (size) {
6737                                         r->mailbox = calloc(1, size);
6738                                         CHECK(r->mailbox, ENOMEM);
6739                                 }
6740
6741                                 /* r->key. */
6742                                 r->key = table->is_header ?
6743                                         &t->structs[table->header->struct_id] :
6744                                         &t->structs[p->metadata_struct_id];
6745                         } else {
6746                                 r->func = table_stub_lkp;
6747                         }
6748                 }
6749         }
6750
6751         return 0;
6752 }
6753
6754 static void
6755 table_build_free(struct rte_swx_pipeline *p)
6756 {
6757         uint32_t i;
6758
6759         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
6760                 struct thread *t = &p->threads[i];
6761                 uint32_t j;
6762
6763                 if (!t->tables)
6764                         continue;
6765
6766                 for (j = 0; j < p->n_tables; j++) {
6767                         struct table_runtime *r = &t->tables[j];
6768
6769                         free(r->mailbox);
6770                 }
6771
6772                 free(t->tables);
6773                 t->tables = NULL;
6774         }
6775 }
6776
6777 static void
6778 table_free(struct rte_swx_pipeline *p)
6779 {
6780         table_build_free(p);
6781
6782         /* Tables. */
6783         for ( ; ; ) {
6784                 struct table *elem;
6785
6786                 elem = TAILQ_FIRST(&p->tables);
6787                 if (!elem)
6788                         break;
6789
6790                 TAILQ_REMOVE(&p->tables, elem, node);
6791                 free(elem->fields);
6792                 free(elem->actions);
6793                 free(elem->default_action_data);
6794                 free(elem);
6795         }
6796
6797         /* Table types. */
6798         for ( ; ; ) {
6799                 struct table_type *elem;
6800
6801                 elem = TAILQ_FIRST(&p->table_types);
6802                 if (!elem)
6803                         break;
6804
6805                 TAILQ_REMOVE(&p->table_types, elem, node);
6806                 free(elem);
6807         }
6808 }
6809
6810 /*
6811  * Pipeline.
6812  */
6813 int
6814 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
6815 {
6816         struct rte_swx_pipeline *pipeline;
6817
6818         /* Check input parameters. */
6819         CHECK(p, EINVAL);
6820
6821         /* Memory allocation. */
6822         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
6823         CHECK(pipeline, ENOMEM);
6824
6825         /* Initialization. */
6826         TAILQ_INIT(&pipeline->struct_types);
6827         TAILQ_INIT(&pipeline->port_in_types);
6828         TAILQ_INIT(&pipeline->ports_in);
6829         TAILQ_INIT(&pipeline->port_out_types);
6830         TAILQ_INIT(&pipeline->ports_out);
6831         TAILQ_INIT(&pipeline->extern_types);
6832         TAILQ_INIT(&pipeline->extern_objs);
6833         TAILQ_INIT(&pipeline->extern_funcs);
6834         TAILQ_INIT(&pipeline->headers);
6835         TAILQ_INIT(&pipeline->actions);
6836         TAILQ_INIT(&pipeline->table_types);
6837         TAILQ_INIT(&pipeline->tables);
6838
6839         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
6840         pipeline->numa_node = numa_node;
6841
6842         *p = pipeline;
6843         return 0;
6844 }
6845
6846 void
6847 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
6848 {
6849         if (!p)
6850                 return;
6851
6852         free(p->instructions);
6853
6854         table_state_free(p);
6855         table_free(p);
6856         action_free(p);
6857         metadata_free(p);
6858         header_free(p);
6859         extern_func_free(p);
6860         extern_obj_free(p);
6861         port_out_free(p);
6862         port_in_free(p);
6863         struct_free(p);
6864
6865         free(p);
6866 }
6867
6868 int
6869 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
6870                                      const char **instructions,
6871                                      uint32_t n_instructions)
6872 {
6873         int err;
6874         uint32_t i;
6875
6876         err = instruction_config(p, NULL, instructions, n_instructions);
6877         if (err)
6878                 return err;
6879
6880         /* Thread instruction pointer reset. */
6881         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
6882                 struct thread *t = &p->threads[i];
6883
6884                 thread_ip_reset(p, t);
6885         }
6886
6887         return 0;
6888 }
6889
6890 int
6891 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
6892 {
6893         int status;
6894
6895         CHECK(p, EINVAL);
6896         CHECK(p->build_done == 0, EEXIST);
6897
6898         status = port_in_build(p);
6899         if (status)
6900                 goto error;
6901
6902         status = port_out_build(p);
6903         if (status)
6904                 goto error;
6905
6906         status = struct_build(p);
6907         if (status)
6908                 goto error;
6909
6910         status = extern_obj_build(p);
6911         if (status)
6912                 goto error;
6913
6914         status = extern_func_build(p);
6915         if (status)
6916                 goto error;
6917
6918         status = header_build(p);
6919         if (status)
6920                 goto error;
6921
6922         status = metadata_build(p);
6923         if (status)
6924                 goto error;
6925
6926         status = action_build(p);
6927         if (status)
6928                 goto error;
6929
6930         status = table_build(p);
6931         if (status)
6932                 goto error;
6933
6934         status = table_state_build(p);
6935         if (status)
6936                 goto error;
6937
6938         p->build_done = 1;
6939         return 0;
6940
6941 error:
6942         table_state_build_free(p);
6943         table_build_free(p);
6944         action_build_free(p);
6945         metadata_build_free(p);
6946         header_build_free(p);
6947         extern_func_build_free(p);
6948         extern_obj_build_free(p);
6949         port_out_build_free(p);
6950         port_in_build_free(p);
6951         struct_build_free(p);
6952
6953         return status;
6954 }
6955
6956 void
6957 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
6958 {
6959         uint32_t i;
6960
6961         for (i = 0; i < n_instructions; i++)
6962                 instr_exec(p);
6963 }
6964
6965 void
6966 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
6967 {
6968         uint32_t i;
6969
6970         for (i = 0; i < p->n_ports_out; i++) {
6971                 struct port_out_runtime *port = &p->out[i];
6972
6973                 if (port->flush)
6974                         port->flush(port->obj);
6975         }
6976 }
6977
6978 /*
6979  * Control.
6980  */
6981 int
6982 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
6983                               struct rte_swx_ctl_pipeline_info *pipeline)
6984 {
6985         struct action *action;
6986         struct table *table;
6987         uint32_t n_actions = 0, n_tables = 0;
6988
6989         if (!p || !pipeline)
6990                 return -EINVAL;
6991
6992         TAILQ_FOREACH(action, &p->actions, node)
6993                 n_actions++;
6994
6995         TAILQ_FOREACH(table, &p->tables, node)
6996                 n_tables++;
6997
6998         pipeline->n_ports_in = p->n_ports_in;
6999         pipeline->n_ports_out = p->n_ports_out;
7000         pipeline->n_actions = n_actions;
7001         pipeline->n_tables = n_tables;
7002
7003         return 0;
7004 }
7005
7006 int
7007 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
7008 {
7009         if (!p || !numa_node)
7010                 return -EINVAL;
7011
7012         *numa_node = p->numa_node;
7013         return 0;
7014 }
7015
7016 int
7017 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
7018                             uint32_t action_id,
7019                             struct rte_swx_ctl_action_info *action)
7020 {
7021         struct action *a = NULL;
7022
7023         if (!p || (action_id >= p->n_actions) || !action)
7024                 return -EINVAL;
7025
7026         a = action_find_by_id(p, action_id);
7027         if (!a)
7028                 return -EINVAL;
7029
7030         strcpy(action->name, a->name);
7031         action->n_args = a->st ? a->st->n_fields : 0;
7032         return 0;
7033 }
7034
7035 int
7036 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
7037                                 uint32_t action_id,
7038                                 uint32_t action_arg_id,
7039                                 struct rte_swx_ctl_action_arg_info *action_arg)
7040 {
7041         struct action *a = NULL;
7042         struct field *arg = NULL;
7043
7044         if (!p || (action_id >= p->n_actions) || !action_arg)
7045                 return -EINVAL;
7046
7047         a = action_find_by_id(p, action_id);
7048         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
7049                 return -EINVAL;
7050
7051         arg = &a->st->fields[action_arg_id];
7052         strcpy(action_arg->name, arg->name);
7053         action_arg->n_bits = arg->n_bits;
7054
7055         return 0;
7056 }
7057
7058 int
7059 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
7060                            uint32_t table_id,
7061                            struct rte_swx_ctl_table_info *table)
7062 {
7063         struct table *t = NULL;
7064
7065         if (!p || !table)
7066                 return -EINVAL;
7067
7068         t = table_find_by_id(p, table_id);
7069         if (!t)
7070                 return -EINVAL;
7071
7072         strcpy(table->name, t->name);
7073         strcpy(table->args, t->args);
7074         table->n_match_fields = t->n_fields;
7075         table->n_actions = t->n_actions;
7076         table->default_action_is_const = t->default_action_is_const;
7077         table->size = t->size;
7078         return 0;
7079 }
7080
7081 int
7082 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
7083         uint32_t table_id,
7084         uint32_t match_field_id,
7085         struct rte_swx_ctl_table_match_field_info *match_field)
7086 {
7087         struct table *t;
7088         struct match_field *f;
7089
7090         if (!p || (table_id >= p->n_tables) || !match_field)
7091                 return -EINVAL;
7092
7093         t = table_find_by_id(p, table_id);
7094         if (!t || (match_field_id >= t->n_fields))
7095                 return -EINVAL;
7096
7097         f = &t->fields[match_field_id];
7098         match_field->match_type = f->match_type;
7099         match_field->is_header = t->is_header;
7100         match_field->n_bits = f->field->n_bits;
7101         match_field->offset = f->field->offset;
7102
7103         return 0;
7104 }
7105
7106 int
7107 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
7108         uint32_t table_id,
7109         uint32_t table_action_id,
7110         struct rte_swx_ctl_table_action_info *table_action)
7111 {
7112         struct table *t;
7113
7114         if (!p || (table_id >= p->n_tables) || !table_action)
7115                 return -EINVAL;
7116
7117         t = table_find_by_id(p, table_id);
7118         if (!t || (table_action_id >= t->n_actions))
7119                 return -EINVAL;
7120
7121         table_action->action_id = t->actions[table_action_id]->id;
7122
7123         return 0;
7124 }
7125
7126 int
7127 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
7128                           uint32_t table_id,
7129                           struct rte_swx_table_ops *table_ops,
7130                           int *is_stub)
7131 {
7132         struct table *t;
7133
7134         if (!p || (table_id >= p->n_tables))
7135                 return -EINVAL;
7136
7137         t = table_find_by_id(p, table_id);
7138         if (!t)
7139                 return -EINVAL;
7140
7141         if (t->type) {
7142                 if (table_ops)
7143                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
7144                 *is_stub = 0;
7145         } else {
7146                 *is_stub = 1;
7147         }
7148
7149         return 0;
7150 }
7151
7152 int
7153 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
7154                                  struct rte_swx_table_state **table_state)
7155 {
7156         if (!p || !table_state || !p->build_done)
7157                 return -EINVAL;
7158
7159         *table_state = p->table_state;
7160         return 0;
7161 }
7162
7163 int
7164 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
7165                                  struct rte_swx_table_state *table_state)
7166 {
7167         if (!p || !table_state || !p->build_done)
7168                 return -EINVAL;
7169
7170         p->table_state = table_state;
7171         return 0;
7172 }
7173
7174 int
7175 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
7176                                         uint32_t port_id,
7177                                         struct rte_swx_port_in_stats *stats)
7178 {
7179         struct port_in *port;
7180
7181         if (!p || !stats)
7182                 return -EINVAL;
7183
7184         port = port_in_find(p, port_id);
7185         if (!port)
7186                 return -EINVAL;
7187
7188         port->type->ops.stats_read(port->obj, stats);
7189         return 0;
7190 }
7191
7192 int
7193 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
7194                                          uint32_t port_id,
7195                                          struct rte_swx_port_out_stats *stats)
7196 {
7197         struct port_out *port;
7198
7199         if (!p || !stats)
7200                 return -EINVAL;
7201
7202         port = port_out_find(p, port_id);
7203         if (!port)
7204                 return -EINVAL;
7205
7206         port->type->ops.stats_read(port->obj, stats);
7207         return 0;
7208 }