lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <inttypes.h>
   9 #include <sys/queue.h>
  10 #include <arpa/inet.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_prefetch.h>
  14 #include <rte_byteorder.h>
  15 #include <rte_cycles.h>
  16 #include <rte_meter.h>
  17
  18 #include <rte_swx_table_selector.h>
  19
  20 #include "rte_swx_pipeline.h"
  21 #include "rte_swx_ctl.h"
  22
  23 #define CHECK(condition, err_code)                                             \
  24 do {                                                                           \
  25         if (!(condition))                                                      \
  26                 return -(err_code);                                            \
  27 } while (0)
  28
  29 #define CHECK_NAME(name, err_code)                                             \
  30         CHECK((name) &&                                                        \
  31               (name)[0] &&                                                     \
  32               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  33               err_code)
  34
  35 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  36         CHECK((instr) &&                                                       \
  37               (instr)[0] &&                                                    \
  38               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  39                RTE_SWX_INSTRUCTION_SIZE),                                      \
  40               err_code)
  41
  42 #ifndef TRACE_LEVEL
  43 #define TRACE_LEVEL 0
  44 #endif
  45
  46 #if TRACE_LEVEL
  47 #define TRACE(...) printf(__VA_ARGS__)
  48 #else
  49 #define TRACE(...)
  50 #endif
  51
  52 /*
  53  * Environment.
  54  */
  55 #define ntoh64(x) rte_be_to_cpu_64(x)
  56 #define hton64(x) rte_cpu_to_be_64(x)
  57
  58 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  59
  60 #include <rte_malloc.h>
  61
  62 static void *
  63 env_malloc(size_t size, size_t alignment, int numa_node)
  64 {
  65         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  66 }
  67
  68 static void
  69 env_free(void *start, size_t size __rte_unused)
  70 {
  71         rte_free(start);
  72 }
  73
  74 #else
  75
  76 #include <numa.h>
  77
  78 static void *
  79 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  80 {
  81         void *start;
  82
  83         if (numa_available() == -1)
  84                 return NULL;
  85
  86         start = numa_alloc_onnode(size, numa_node);
  87         if (!start)
  88                 return NULL;
  89
  90         memset(start, 0, size);
  91         return start;
  92 }
  93
  94 static void
  95 env_free(void *start, size_t size)
  96 {
  97         if (numa_available() == -1)
  98                 return;
  99
 100         numa_free(start, size);
 101 }
 102
 103 #endif
 104
 105 /*
 106  * Struct.
 107  */
 108 struct field {
 109         char name[RTE_SWX_NAME_SIZE];
 110         uint32_t n_bits;
 111         uint32_t offset;
 112         int var_size;
 113 };
 114
 115 struct struct_type {
 116         TAILQ_ENTRY(struct_type) node;
 117         char name[RTE_SWX_NAME_SIZE];
 118         struct field *fields;
 119         uint32_t n_fields;
 120         uint32_t n_bits;
 121         uint32_t n_bits_min;
 122         int var_size;
 123 };
 124
 125 TAILQ_HEAD(struct_type_tailq, struct_type);
 126
 127 /*
 128  * Input port.
 129  */
 130 struct port_in_type {
 131         TAILQ_ENTRY(port_in_type) node;
 132         char name[RTE_SWX_NAME_SIZE];
 133         struct rte_swx_port_in_ops ops;
 134 };
 135
 136 TAILQ_HEAD(port_in_type_tailq, port_in_type);
 137
 138 struct port_in {
 139         TAILQ_ENTRY(port_in) node;
 140         struct port_in_type *type;
 141         void *obj;
 142         uint32_t id;
 143 };
 144
 145 TAILQ_HEAD(port_in_tailq, port_in);
 146
 147 struct port_in_runtime {
 148         rte_swx_port_in_pkt_rx_t pkt_rx;
 149         void *obj;
 150 };
 151
 152 /*
 153  * Output port.
 154  */
 155 struct port_out_type {
 156         TAILQ_ENTRY(port_out_type) node;
 157         char name[RTE_SWX_NAME_SIZE];
 158         struct rte_swx_port_out_ops ops;
 159 };
 160
 161 TAILQ_HEAD(port_out_type_tailq, port_out_type);
 162
 163 struct port_out {
 164         TAILQ_ENTRY(port_out) node;
 165         struct port_out_type *type;
 166         void *obj;
 167         uint32_t id;
 168 };
 169
 170 TAILQ_HEAD(port_out_tailq, port_out);
 171
 172 struct port_out_runtime {
 173         rte_swx_port_out_pkt_tx_t pkt_tx;
 174         rte_swx_port_out_flush_t flush;
 175         void *obj;
 176 };
 177
 178 /*
 179  * Extern object.
 180  */
 181 struct extern_type_member_func {
 182         TAILQ_ENTRY(extern_type_member_func) node;
 183         char name[RTE_SWX_NAME_SIZE];
 184         rte_swx_extern_type_member_func_t func;
 185         uint32_t id;
 186 };
 187
 188 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 189
 190 struct extern_type {
 191         TAILQ_ENTRY(extern_type) node;
 192         char name[RTE_SWX_NAME_SIZE];
 193         struct struct_type *mailbox_struct_type;
 194         rte_swx_extern_type_constructor_t constructor;
 195         rte_swx_extern_type_destructor_t destructor;
 196         struct extern_type_member_func_tailq funcs;
 197         uint32_t n_funcs;
 198 };
 199
 200 TAILQ_HEAD(extern_type_tailq, extern_type);
 201
 202 struct extern_obj {
 203         TAILQ_ENTRY(extern_obj) node;
 204         char name[RTE_SWX_NAME_SIZE];
 205         struct extern_type *type;
 206         void *obj;
 207         uint32_t struct_id;
 208         uint32_t id;
 209 };
 210
 211 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 212
 213 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 214 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 215 #endif
 216
 217 struct extern_obj_runtime {
 218         void *obj;
 219         uint8_t *mailbox;
 220         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 221 };
 222
 223 /*
 224  * Extern function.
 225  */
 226 struct extern_func {
 227         TAILQ_ENTRY(extern_func) node;
 228         char name[RTE_SWX_NAME_SIZE];
 229         struct struct_type *mailbox_struct_type;
 230         rte_swx_extern_func_t func;
 231         uint32_t struct_id;
 232         uint32_t id;
 233 };
 234
 235 TAILQ_HEAD(extern_func_tailq, extern_func);
 236
 237 struct extern_func_runtime {
 238         uint8_t *mailbox;
 239         rte_swx_extern_func_t func;
 240 };
 241
 242 /*
 243  * Header.
 244  */
 245 struct header {
 246         TAILQ_ENTRY(header) node;
 247         char name[RTE_SWX_NAME_SIZE];
 248         struct struct_type *st;
 249         uint32_t struct_id;
 250         uint32_t id;
 251 };
 252
 253 TAILQ_HEAD(header_tailq, header);
 254
 255 struct header_runtime {
 256         uint8_t *ptr0;
 257         uint32_t n_bytes;
 258 };
 259
 260 struct header_out_runtime {
 261         uint8_t *ptr0;
 262         uint8_t *ptr;
 263         uint32_t n_bytes;
 264 };
 265
 266 /*
 267  * Instruction.
 268  */
 269
 270 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 271  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 272  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 273  * when transferred to packet meta-data and in NBO when transferred to packet
 274  * headers.
 275  */
 276
 277 /* Notation conventions:
 278  *    -Header field: H = h.header.field (dst/src)
 279  *    -Meta-data field: M = m.field (dst/src)
 280  *    -Extern object mailbox field: E = e.field (dst/src)
 281  *    -Extern function mailbox field: F = f.field (dst/src)
 282  *    -Table action data field: T = t.field (src only)
 283  *    -Immediate value: I = 32-bit unsigned value (src only)
 284  */
 285
 286 enum instruction_type {
 287         /* rx m.port_in */
 288         INSTR_RX,
 289
 290         /* tx port_out
 291          * port_out = MI
 292          */
 293         INSTR_TX,   /* port_out = M */
 294         INSTR_TX_I, /* port_out = I */
 295
 296         /* extract h.header */
 297         INSTR_HDR_EXTRACT,
 298         INSTR_HDR_EXTRACT2,
 299         INSTR_HDR_EXTRACT3,
 300         INSTR_HDR_EXTRACT4,
 301         INSTR_HDR_EXTRACT5,
 302         INSTR_HDR_EXTRACT6,
 303         INSTR_HDR_EXTRACT7,
 304         INSTR_HDR_EXTRACT8,
 305
 306         /* extract h.header m.last_field_size */
 307         INSTR_HDR_EXTRACT_M,
 308
 309         /* emit h.header */
 310         INSTR_HDR_EMIT,
 311         INSTR_HDR_EMIT_TX,
 312         INSTR_HDR_EMIT2_TX,
 313         INSTR_HDR_EMIT3_TX,
 314         INSTR_HDR_EMIT4_TX,
 315         INSTR_HDR_EMIT5_TX,
 316         INSTR_HDR_EMIT6_TX,
 317         INSTR_HDR_EMIT7_TX,
 318         INSTR_HDR_EMIT8_TX,
 319
 320         /* validate h.header */
 321         INSTR_HDR_VALIDATE,
 322
 323         /* invalidate h.header */
 324         INSTR_HDR_INVALIDATE,
 325
 326         /* mov dst src
 327          * dst = src
 328          * dst = HMEF, src = HMEFTI
 329          */
 330         INSTR_MOV,    /* dst = MEF, src = MEFT */
 331         INSTR_MOV_MH, /* dst = MEF, src = H */
 332         INSTR_MOV_HM, /* dst = H, src = MEFT */
 333         INSTR_MOV_HH, /* dst = H, src = H */
 334         INSTR_MOV_I,  /* dst = HMEF, src = I */
 335
 336         /* dma h.header t.field
 337          * memcpy(h.header, t.field, sizeof(h.header))
 338          */
 339         INSTR_DMA_HT,
 340         INSTR_DMA_HT2,
 341         INSTR_DMA_HT3,
 342         INSTR_DMA_HT4,
 343         INSTR_DMA_HT5,
 344         INSTR_DMA_HT6,
 345         INSTR_DMA_HT7,
 346         INSTR_DMA_HT8,
 347
 348         /* add dst src
 349          * dst += src
 350          * dst = HMEF, src = HMEFTI
 351          */
 352         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 353         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 354         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 355         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 356         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 357         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 358
 359         /* sub dst src
 360          * dst -= src
 361          * dst = HMEF, src = HMEFTI
 362          */
 363         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 364         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 365         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 366         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 367         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 368         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 369
 370         /* ckadd dst src
 371          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 372          * dst = H, src = {H, h.header}
 373          */
 374         INSTR_ALU_CKADD_FIELD,    /* src = H */
 375         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 376         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 377
 378         /* cksub dst src
 379          * dst = dst '- src
 380          * dst = H, src = H
 381          */
 382         INSTR_ALU_CKSUB_FIELD,
 383
 384         /* and dst src
 385          * dst &= src
 386          * dst = HMEF, src = HMEFTI
 387          */
 388         INSTR_ALU_AND,    /* dst = MEF, src = MEFT */
 389         INSTR_ALU_AND_MH, /* dst = MEF, src = H */
 390         INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
 391         INSTR_ALU_AND_HH, /* dst = H, src = H */
 392         INSTR_ALU_AND_I,  /* dst = HMEF, src = I */
 393
 394         /* or dst src
 395          * dst |= src
 396          * dst = HMEF, src = HMEFTI
 397          */
 398         INSTR_ALU_OR,    /* dst = MEF, src = MEFT */
 399         INSTR_ALU_OR_MH, /* dst = MEF, src = H */
 400         INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
 401         INSTR_ALU_OR_HH, /* dst = H, src = H */
 402         INSTR_ALU_OR_I,  /* dst = HMEF, src = I */
 403
 404         /* xor dst src
 405          * dst ^= src
 406          * dst = HMEF, src = HMEFTI
 407          */
 408         INSTR_ALU_XOR,    /* dst = MEF, src = MEFT */
 409         INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
 410         INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
 411         INSTR_ALU_XOR_HH, /* dst = H, src = H */
 412         INSTR_ALU_XOR_I,  /* dst = HMEF, src = I */
 413
 414         /* shl dst src
 415          * dst <<= src
 416          * dst = HMEF, src = HMEFTI
 417          */
 418         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 419         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 420         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 421         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 422         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 423         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 424
 425         /* shr dst src
 426          * dst >>= src
 427          * dst = HMEF, src = HMEFTI
 428          */
 429         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 430         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 431         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 432         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 433         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 434         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 435
 436         /* regprefetch REGARRAY index
 437          * prefetch REGARRAY[index]
 438          * index = HMEFTI
 439          */
 440         INSTR_REGPREFETCH_RH, /* index = H */
 441         INSTR_REGPREFETCH_RM, /* index = MEFT */
 442         INSTR_REGPREFETCH_RI, /* index = I */
 443
 444         /* regrd dst REGARRAY index
 445          * dst = REGARRAY[index]
 446          * dst = HMEF, index = HMEFTI
 447          */
 448         INSTR_REGRD_HRH, /* dst = H, index = H */
 449         INSTR_REGRD_HRM, /* dst = H, index = MEFT */
 450         INSTR_REGRD_HRI, /* dst = H, index = I */
 451         INSTR_REGRD_MRH, /* dst = MEF, index = H */
 452         INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
 453         INSTR_REGRD_MRI, /* dst = MEF, index = I */
 454
 455         /* regwr REGARRAY index src
 456          * REGARRAY[index] = src
 457          * index = HMEFTI, src = HMEFTI
 458          */
 459         INSTR_REGWR_RHH, /* index = H, src = H */
 460         INSTR_REGWR_RHM, /* index = H, src = MEFT */
 461         INSTR_REGWR_RHI, /* index = H, src = I */
 462         INSTR_REGWR_RMH, /* index = MEFT, src = H */
 463         INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
 464         INSTR_REGWR_RMI, /* index = MEFT, src = I */
 465         INSTR_REGWR_RIH, /* index = I, src = H */
 466         INSTR_REGWR_RIM, /* index = I, src = MEFT */
 467         INSTR_REGWR_RII, /* index = I, src = I */
 468
 469         /* regadd REGARRAY index src
 470          * REGARRAY[index] += src
 471          * index = HMEFTI, src = HMEFTI
 472          */
 473         INSTR_REGADD_RHH, /* index = H, src = H */
 474         INSTR_REGADD_RHM, /* index = H, src = MEFT */
 475         INSTR_REGADD_RHI, /* index = H, src = I */
 476         INSTR_REGADD_RMH, /* index = MEFT, src = H */
 477         INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
 478         INSTR_REGADD_RMI, /* index = MEFT, src = I */
 479         INSTR_REGADD_RIH, /* index = I, src = H */
 480         INSTR_REGADD_RIM, /* index = I, src = MEFT */
 481         INSTR_REGADD_RII, /* index = I, src = I */
 482
 483         /* metprefetch METARRAY index
 484          * prefetch METARRAY[index]
 485          * index = HMEFTI
 486          */
 487         INSTR_METPREFETCH_H, /* index = H */
 488         INSTR_METPREFETCH_M, /* index = MEFT */
 489         INSTR_METPREFETCH_I, /* index = I */
 490
 491         /* meter METARRAY index length color_in color_out
 492          * color_out = meter(METARRAY[index], length, color_in)
 493          * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
 494          */
 495         INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
 496         INSTR_METER_HHI, /* index = H, length = H, color_in = I */
 497         INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
 498         INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
 499         INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
 500         INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
 501         INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
 502         INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
 503         INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
 504         INSTR_METER_IHI, /* index = I, length = H, color_in = I */
 505         INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
 506         INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
 507
 508         /* table TABLE */
 509         INSTR_TABLE,
 510         INSTR_SELECTOR,
 511
 512         /* extern e.obj.func */
 513         INSTR_EXTERN_OBJ,
 514
 515         /* extern f.func */
 516         INSTR_EXTERN_FUNC,
 517
 518         /* jmp LABEL
 519          * Unconditional jump
 520          */
 521         INSTR_JMP,
 522
 523         /* jmpv LABEL h.header
 524          * Jump if header is valid
 525          */
 526         INSTR_JMP_VALID,
 527
 528         /* jmpnv LABEL h.header
 529          * Jump if header is invalid
 530          */
 531         INSTR_JMP_INVALID,
 532
 533         /* jmph LABEL
 534          * Jump if table lookup hit
 535          */
 536         INSTR_JMP_HIT,
 537
 538         /* jmpnh LABEL
 539          * Jump if table lookup miss
 540          */
 541         INSTR_JMP_MISS,
 542
 543         /* jmpa LABEL ACTION
 544          * Jump if action run
 545          */
 546         INSTR_JMP_ACTION_HIT,
 547
 548         /* jmpna LABEL ACTION
 549          * Jump if action not run
 550          */
 551         INSTR_JMP_ACTION_MISS,
 552
 553         /* jmpeq LABEL a b
 554          * Jump if a is equal to b
 555          * a = HMEFT, b = HMEFTI
 556          */
 557         INSTR_JMP_EQ,    /* a = MEFT, b = MEFT */
 558         INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
 559         INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
 560         INSTR_JMP_EQ_HH, /* a = H, b = H */
 561         INSTR_JMP_EQ_I,  /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 562
 563         /* jmpneq LABEL a b
 564          * Jump if a is not equal to b
 565          * a = HMEFT, b = HMEFTI
 566          */
 567         INSTR_JMP_NEQ,    /* a = MEFT, b = MEFT */
 568         INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
 569         INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
 570         INSTR_JMP_NEQ_HH, /* a = H, b = H */
 571         INSTR_JMP_NEQ_I,  /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 572
 573         /* jmplt LABEL a b
 574          * Jump if a is less than b
 575          * a = HMEFT, b = HMEFTI
 576          */
 577         INSTR_JMP_LT,    /* a = MEFT, b = MEFT */
 578         INSTR_JMP_LT_MH, /* a = MEFT, b = H */
 579         INSTR_JMP_LT_HM, /* a = H, b = MEFT */
 580         INSTR_JMP_LT_HH, /* a = H, b = H */
 581         INSTR_JMP_LT_MI, /* a = MEFT, b = I */
 582         INSTR_JMP_LT_HI, /* a = H, b = I */
 583
 584         /* jmpgt LABEL a b
 585          * Jump if a is greater than b
 586          * a = HMEFT, b = HMEFTI
 587          */
 588         INSTR_JMP_GT,    /* a = MEFT, b = MEFT */
 589         INSTR_JMP_GT_MH, /* a = MEFT, b = H */
 590         INSTR_JMP_GT_HM, /* a = H, b = MEFT */
 591         INSTR_JMP_GT_HH, /* a = H, b = H */
 592         INSTR_JMP_GT_MI, /* a = MEFT, b = I */
 593         INSTR_JMP_GT_HI, /* a = H, b = I */
 594
 595         /* return
 596          * Return from action
 597          */
 598         INSTR_RETURN,
 599 };
 600
 601 struct instr_operand {
 602         uint8_t struct_id;
 603         uint8_t n_bits;
 604         uint8_t offset;
 605         uint8_t pad;
 606 };
 607
 608 struct instr_io {
 609         struct {
 610                 union {
 611                         struct {
 612                                 uint8_t offset;
 613                                 uint8_t n_bits;
 614                                 uint8_t pad[2];
 615                         };
 616
 617                         uint32_t val;
 618                 };
 619         } io;
 620
 621         struct {
 622                 uint8_t header_id[8];
 623                 uint8_t struct_id[8];
 624                 uint8_t n_bytes[8];
 625         } hdr;
 626 };
 627
 628 struct instr_hdr_validity {
 629         uint8_t header_id;
 630 };
 631
 632 struct instr_table {
 633         uint8_t table_id;
 634 };
 635
 636 struct instr_extern_obj {
 637         uint8_t ext_obj_id;
 638         uint8_t func_id;
 639 };
 640
 641 struct instr_extern_func {
 642         uint8_t ext_func_id;
 643 };
 644
 645 struct instr_dst_src {
 646         struct instr_operand dst;
 647         union {
 648                 struct instr_operand src;
 649                 uint64_t src_val;
 650         };
 651 };
 652
 653 struct instr_regarray {
 654         uint8_t regarray_id;
 655         uint8_t pad[3];
 656
 657         union {
 658                 struct instr_operand idx;
 659                 uint32_t idx_val;
 660         };
 661
 662         union {
 663                 struct instr_operand dstsrc;
 664                 uint64_t dstsrc_val;
 665         };
 666 };
 667
 668 struct instr_meter {
 669         uint8_t metarray_id;
 670         uint8_t pad[3];
 671
 672         union {
 673                 struct instr_operand idx;
 674                 uint32_t idx_val;
 675         };
 676
 677         struct instr_operand length;
 678
 679         union {
 680                 struct instr_operand color_in;
 681                 uint32_t color_in_val;
 682         };
 683
 684         struct instr_operand color_out;
 685 };
 686
 687 struct instr_dma {
 688         struct {
 689                 uint8_t header_id[8];
 690                 uint8_t struct_id[8];
 691         } dst;
 692
 693         struct {
 694                 uint8_t offset[8];
 695         } src;
 696
 697         uint16_t n_bytes[8];
 698 };
 699
 700 struct instr_jmp {
 701         struct instruction *ip;
 702
 703         union {
 704                 struct instr_operand a;
 705                 uint8_t header_id;
 706                 uint8_t action_id;
 707         };
 708
 709         union {
 710                 struct instr_operand b;
 711                 uint64_t b_val;
 712         };
 713 };
 714
 715 struct instruction {
 716         enum instruction_type type;
 717         union {
 718                 struct instr_io io;
 719                 struct instr_hdr_validity valid;
 720                 struct instr_dst_src mov;
 721                 struct instr_regarray regarray;
 722                 struct instr_meter meter;
 723                 struct instr_dma dma;
 724                 struct instr_dst_src alu;
 725                 struct instr_table table;
 726                 struct instr_extern_obj ext_obj;
 727                 struct instr_extern_func ext_func;
 728                 struct instr_jmp jmp;
 729         };
 730 };
 731
 732 struct instruction_data {
 733         char label[RTE_SWX_NAME_SIZE];
 734         char jmp_label[RTE_SWX_NAME_SIZE];
 735         uint32_t n_users; /* user = jmp instruction to this instruction. */
 736         int invalid;
 737 };
 738
 739 /*
 740  * Action.
 741  */
 742 struct action {
 743         TAILQ_ENTRY(action) node;
 744         char name[RTE_SWX_NAME_SIZE];
 745         struct struct_type *st;
 746         int *args_endianness; /* 0 = Host Byte Order (HBO). */
 747         struct instruction *instructions;
 748         uint32_t n_instructions;
 749         uint32_t id;
 750 };
 751
 752 TAILQ_HEAD(action_tailq, action);
 753
 754 /*
 755  * Table.
 756  */
 757 struct table_type {
 758         TAILQ_ENTRY(table_type) node;
 759         char name[RTE_SWX_NAME_SIZE];
 760         enum rte_swx_table_match_type match_type;
 761         struct rte_swx_table_ops ops;
 762 };
 763
 764 TAILQ_HEAD(table_type_tailq, table_type);
 765
 766 struct match_field {
 767         enum rte_swx_table_match_type match_type;
 768         struct field *field;
 769 };
 770
 771 struct table {
 772         TAILQ_ENTRY(table) node;
 773         char name[RTE_SWX_NAME_SIZE];
 774         char args[RTE_SWX_NAME_SIZE];
 775         struct table_type *type; /* NULL when n_fields == 0. */
 776
 777         /* Match. */
 778         struct match_field *fields;
 779         uint32_t n_fields;
 780         struct header *header; /* Only valid when n_fields > 0. */
 781
 782         /* Action. */
 783         struct action **actions;
 784         struct action *default_action;
 785         uint8_t *default_action_data;
 786         uint32_t n_actions;
 787         int default_action_is_const;
 788         uint32_t action_data_size_max;
 789
 790         uint32_t size;
 791         uint32_t id;
 792 };
 793
 794 TAILQ_HEAD(table_tailq, table);
 795
 796 struct table_runtime {
 797         rte_swx_table_lookup_t func;
 798         void *mailbox;
 799         uint8_t **key;
 800 };
 801
 802 struct table_statistics {
 803         uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
 804         uint64_t *n_pkts_action;
 805 };
 806
 807 /*
 808  * Selector.
 809  */
 810 struct selector {
 811         TAILQ_ENTRY(selector) node;
 812         char name[RTE_SWX_NAME_SIZE];
 813
 814         struct field *group_id_field;
 815         struct field **selector_fields;
 816         uint32_t n_selector_fields;
 817         struct header *selector_header;
 818         struct field *member_id_field;
 819
 820         uint32_t n_groups_max;
 821         uint32_t n_members_per_group_max;
 822
 823         uint32_t id;
 824 };
 825
 826 TAILQ_HEAD(selector_tailq, selector);
 827
 828 struct selector_runtime {
 829         void *mailbox;
 830         uint8_t **group_id_buffer;
 831         uint8_t **selector_buffer;
 832         uint8_t **member_id_buffer;
 833 };
 834
 835 struct selector_statistics {
 836         uint64_t n_pkts;
 837 };
 838
 839 /*
 840  * Register array.
 841  */
 842 struct regarray {
 843         TAILQ_ENTRY(regarray) node;
 844         char name[RTE_SWX_NAME_SIZE];
 845         uint64_t init_val;
 846         uint32_t size;
 847         uint32_t id;
 848 };
 849
 850 TAILQ_HEAD(regarray_tailq, regarray);
 851
 852 struct regarray_runtime {
 853         uint64_t *regarray;
 854         uint32_t size_mask;
 855 };
 856
 857 /*
 858  * Meter array.
 859  */
 860 struct meter_profile {
 861         TAILQ_ENTRY(meter_profile) node;
 862         char name[RTE_SWX_NAME_SIZE];
 863         struct rte_meter_trtcm_params params;
 864         struct rte_meter_trtcm_profile profile;
 865         uint32_t n_users;
 866 };
 867
 868 TAILQ_HEAD(meter_profile_tailq, meter_profile);
 869
 870 struct metarray {
 871         TAILQ_ENTRY(metarray) node;
 872         char name[RTE_SWX_NAME_SIZE];
 873         uint32_t size;
 874         uint32_t id;
 875 };
 876
 877 TAILQ_HEAD(metarray_tailq, metarray);
 878
 879 struct meter {
 880         struct rte_meter_trtcm m;
 881         struct meter_profile *profile;
 882         enum rte_color color_mask;
 883         uint8_t pad[20];
 884
 885         uint64_t n_pkts[RTE_COLORS];
 886         uint64_t n_bytes[RTE_COLORS];
 887 };
 888
 889 struct metarray_runtime {
 890         struct meter *metarray;
 891         uint32_t size_mask;
 892 };
 893
 894 /*
 895  * Pipeline.
 896  */
 897 struct thread {
 898         /* Packet. */
 899         struct rte_swx_pkt pkt;
 900         uint8_t *ptr;
 901
 902         /* Structures. */
 903         uint8_t **structs;
 904
 905         /* Packet headers. */
 906         struct header_runtime *headers; /* Extracted or generated headers. */
 907         struct header_out_runtime *headers_out; /* Emitted headers. */
 908         uint8_t *header_storage;
 909         uint8_t *header_out_storage;
 910         uint64_t valid_headers;
 911         uint32_t n_headers_out;
 912
 913         /* Packet meta-data. */
 914         uint8_t *metadata;
 915
 916         /* Tables. */
 917         struct table_runtime *tables;
 918         struct selector_runtime *selectors;
 919         struct rte_swx_table_state *table_state;
 920         uint64_t action_id;
 921         int hit; /* 0 = Miss, 1 = Hit. */
 922
 923         /* Extern objects and functions. */
 924         struct extern_obj_runtime *extern_objs;
 925         struct extern_func_runtime *extern_funcs;
 926
 927         /* Instructions. */
 928         struct instruction *ip;
 929         struct instruction *ret;
 930 };
 931
 932 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 933 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 934 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 935
 936 #define HEADER_VALID(thread, header_id) \
 937         MASK64_BIT_GET((thread)->valid_headers, header_id)
 938
 939 #define ALU(thread, ip, operator)  \
 940 {                                                                              \
 941         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 942         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 943         uint64_t dst64 = *dst64_ptr;                                           \
 944         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 945         uint64_t dst = dst64 & dst64_mask;                                     \
 946                                                                                \
 947         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 948         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 949         uint64_t src64 = *src64_ptr;                                           \
 950         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 951         uint64_t src = src64 & src64_mask;                                     \
 952                                                                                \
 953         uint64_t result = dst operator src;                                    \
 954                                                                                \
 955         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 956 }
 957
 958 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 959
 960 #define ALU_MH(thread, ip, operator)  \
 961 {                                                                              \
 962         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 963         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 964         uint64_t dst64 = *dst64_ptr;                                           \
 965         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 966         uint64_t dst = dst64 & dst64_mask;                                     \
 967                                                                                \
 968         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 969         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 970         uint64_t src64 = *src64_ptr;                                           \
 971         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 972                                                                                \
 973         uint64_t result = dst operator src;                                    \
 974                                                                                \
 975         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 976 }
 977
 978 #define ALU_HM(thread, ip, operator)  \
 979 {                                                                              \
 980         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 981         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 982         uint64_t dst64 = *dst64_ptr;                                           \
 983         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 984         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 985                                                                                \
 986         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 987         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 988         uint64_t src64 = *src64_ptr;                                           \
 989         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 990         uint64_t src = src64 & src64_mask;                                     \
 991                                                                                \
 992         uint64_t result = dst operator src;                                    \
 993         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 994                                                                                \
 995         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 996 }
 997
 998 #define ALU_HM_FAST(thread, ip, operator)  \
 999 {                                                                                 \
1000         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];         \
1001         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];      \
1002         uint64_t dst64 = *dst64_ptr;                                              \
1003         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);          \
1004         uint64_t dst = dst64 & dst64_mask;                                        \
1005                                                                                   \
1006         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];         \
1007         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];      \
1008         uint64_t src64 = *src64_ptr;                                              \
1009         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);          \
1010         uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1011                                                                                   \
1012         uint64_t result = dst operator src;                                       \
1013                                                                                   \
1014         *dst64_ptr = (dst64 & ~dst64_mask) | result;                              \
1015 }
1016
1017 #define ALU_HH(thread, ip, operator)  \
1018 {                                                                              \
1019         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1020         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1021         uint64_t dst64 = *dst64_ptr;                                           \
1022         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1023         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
1024                                                                                \
1025         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
1026         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
1027         uint64_t src64 = *src64_ptr;                                           \
1028         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
1029                                                                                \
1030         uint64_t result = dst operator src;                                    \
1031         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1032                                                                                \
1033         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1034 }
1035
1036 #define ALU_HH_FAST(thread, ip, operator)  \
1037 {                                                                                             \
1038         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];                     \
1039         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];                  \
1040         uint64_t dst64 = *dst64_ptr;                                                          \
1041         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);                      \
1042         uint64_t dst = dst64 & dst64_mask;                                                    \
1043                                                                                               \
1044         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];                     \
1045         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];                  \
1046         uint64_t src64 = *src64_ptr;                                                          \
1047         uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1048                                                                                               \
1049         uint64_t result = dst operator src;                                                   \
1050                                                                                               \
1051         *dst64_ptr = (dst64 & ~dst64_mask) | result;                                          \
1052 }
1053
1054 #else
1055
1056 #define ALU_MH ALU
1057 #define ALU_HM ALU
1058 #define ALU_HM_FAST ALU
1059 #define ALU_HH ALU
1060 #define ALU_HH_FAST ALU
1061
1062 #endif
1063
1064 #define ALU_I(thread, ip, operator)  \
1065 {                                                                              \
1066         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1067         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1068         uint64_t dst64 = *dst64_ptr;                                           \
1069         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1070         uint64_t dst = dst64 & dst64_mask;                                     \
1071                                                                                \
1072         uint64_t src = (ip)->alu.src_val;                                      \
1073                                                                                \
1074         uint64_t result = dst operator src;                                    \
1075                                                                                \
1076         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
1077 }
1078
1079 #define ALU_MI ALU_I
1080
1081 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1082
1083 #define ALU_HI(thread, ip, operator)  \
1084 {                                                                              \
1085         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1086         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1087         uint64_t dst64 = *dst64_ptr;                                           \
1088         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1089         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
1090                                                                                \
1091         uint64_t src = (ip)->alu.src_val;                                      \
1092                                                                                \
1093         uint64_t result = dst operator src;                                    \
1094         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1095                                                                                \
1096         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1097 }
1098
1099 #else
1100
1101 #define ALU_HI ALU_I
1102
1103 #endif
1104
1105 #define MOV(thread, ip)  \
1106 {                                                                              \
1107         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1108         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1109         uint64_t dst64 = *dst64_ptr;                                           \
1110         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1111                                                                                \
1112         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1113         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1114         uint64_t src64 = *src64_ptr;                                           \
1115         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1116         uint64_t src = src64 & src64_mask;                                     \
1117                                                                                \
1118         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1119 }
1120
1121 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1122
1123 #define MOV_MH(thread, ip)  \
1124 {                                                                              \
1125         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1126         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1127         uint64_t dst64 = *dst64_ptr;                                           \
1128         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1129                                                                                \
1130         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1131         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1132         uint64_t src64 = *src64_ptr;                                           \
1133         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
1134                                                                                \
1135         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1136 }
1137
1138 #define MOV_HM(thread, ip)  \
1139 {                                                                              \
1140         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1141         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1142         uint64_t dst64 = *dst64_ptr;                                           \
1143         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1144                                                                                \
1145         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1146         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1147         uint64_t src64 = *src64_ptr;                                           \
1148         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1149         uint64_t src = src64 & src64_mask;                                     \
1150                                                                                \
1151         src = hton64(src) >> (64 - (ip)->mov.dst.n_bits);                      \
1152         *dst64_ptr = (dst64 & ~dst64_mask) | src;                              \
1153 }
1154
1155 #define MOV_HH(thread, ip)  \
1156 {                                                                              \
1157         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1158         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1159         uint64_t dst64 = *dst64_ptr;                                           \
1160         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1161                                                                                \
1162         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1163         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1164         uint64_t src64 = *src64_ptr;                                           \
1165                                                                                \
1166         uint64_t src = src64 << (64 - (ip)->mov.src.n_bits);                   \
1167         src = src >> (64 - (ip)->mov.dst.n_bits);                              \
1168         *dst64_ptr = (dst64 & ~dst64_mask) | src;                              \
1169 }
1170
1171 #else
1172
1173 #define MOV_MH MOV
1174 #define MOV_HM MOV
1175 #define MOV_HH MOV
1176
1177 #endif
1178
1179 #define MOV_I(thread, ip)  \
1180 {                                                                              \
1181         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1182         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1183         uint64_t dst64 = *dst64_ptr;                                           \
1184         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1185                                                                                \
1186         uint64_t src = (ip)->mov.src_val;                                      \
1187                                                                                \
1188         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1189 }
1190
1191 #define JMP_CMP(thread, ip, operator)  \
1192 {                                                                              \
1193         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1194         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1195         uint64_t a64 = *a64_ptr;                                               \
1196         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1197         uint64_t a = a64 & a64_mask;                                           \
1198                                                                                \
1199         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1200         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1201         uint64_t b64 = *b64_ptr;                                               \
1202         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1203         uint64_t b = b64 & b64_mask;                                           \
1204                                                                                \
1205         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1206 }
1207
1208 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1209
1210 #define JMP_CMP_MH(thread, ip, operator)  \
1211 {                                                                              \
1212         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1213         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1214         uint64_t a64 = *a64_ptr;                                               \
1215         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1216         uint64_t a = a64 & a64_mask;                                           \
1217                                                                                \
1218         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1219         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1220         uint64_t b64 = *b64_ptr;                                               \
1221         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1222                                                                                \
1223         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1224 }
1225
1226 #define JMP_CMP_HM(thread, ip, operator)  \
1227 {                                                                              \
1228         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1229         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1230         uint64_t a64 = *a64_ptr;                                               \
1231         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1232                                                                                \
1233         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1234         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1235         uint64_t b64 = *b64_ptr;                                               \
1236         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1237         uint64_t b = b64 & b64_mask;                                           \
1238                                                                                \
1239         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1240 }
1241
1242 #define JMP_CMP_HH(thread, ip, operator)  \
1243 {                                                                              \
1244         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1245         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1246         uint64_t a64 = *a64_ptr;                                               \
1247         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1248                                                                                \
1249         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1250         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1251         uint64_t b64 = *b64_ptr;                                               \
1252         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1253                                                                                \
1254         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1255 }
1256
1257 #define JMP_CMP_HH_FAST(thread, ip, operator)  \
1258 {                                                                              \
1259         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1260         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1261         uint64_t a64 = *a64_ptr;                                               \
1262         uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits);                         \
1263                                                                                \
1264         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1265         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1266         uint64_t b64 = *b64_ptr;                                               \
1267         uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits);                         \
1268                                                                                \
1269         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1270 }
1271
1272 #else
1273
1274 #define JMP_CMP_MH JMP_CMP
1275 #define JMP_CMP_HM JMP_CMP
1276 #define JMP_CMP_HH JMP_CMP
1277 #define JMP_CMP_HH_FAST JMP_CMP
1278
1279 #endif
1280
1281 #define JMP_CMP_I(thread, ip, operator)  \
1282 {                                                                              \
1283         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1284         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1285         uint64_t a64 = *a64_ptr;                                               \
1286         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1287         uint64_t a = a64 & a64_mask;                                           \
1288                                                                                \
1289         uint64_t b = (ip)->jmp.b_val;                                          \
1290                                                                                \
1291         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1292 }
1293
1294 #define JMP_CMP_MI JMP_CMP_I
1295
1296 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1297
1298 #define JMP_CMP_HI(thread, ip, operator)  \
1299 {                                                                              \
1300         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1301         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1302         uint64_t a64 = *a64_ptr;                                               \
1303         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1304                                                                                \
1305         uint64_t b = (ip)->jmp.b_val;                                          \
1306                                                                                \
1307         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1308 }
1309
1310 #else
1311
1312 #define JMP_CMP_HI JMP_CMP_I
1313
1314 #endif
1315
1316 #define METADATA_READ(thread, offset, n_bits)                                  \
1317 ({                                                                             \
1318         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1319         uint64_t m64 = *m64_ptr;                                               \
1320         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1321         (m64 & m64_mask);                                                      \
1322 })
1323
1324 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
1325 {                                                                              \
1326         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1327         uint64_t m64 = *m64_ptr;                                               \
1328         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1329                                                                                \
1330         uint64_t m_new = value;                                                \
1331                                                                                \
1332         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
1333 }
1334
1335 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1336 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1337 #endif
1338
1339 struct rte_swx_pipeline {
1340         struct struct_type_tailq struct_types;
1341         struct port_in_type_tailq port_in_types;
1342         struct port_in_tailq ports_in;
1343         struct port_out_type_tailq port_out_types;
1344         struct port_out_tailq ports_out;
1345         struct extern_type_tailq extern_types;
1346         struct extern_obj_tailq extern_objs;
1347         struct extern_func_tailq extern_funcs;
1348         struct header_tailq headers;
1349         struct struct_type *metadata_st;
1350         uint32_t metadata_struct_id;
1351         struct action_tailq actions;
1352         struct table_type_tailq table_types;
1353         struct table_tailq tables;
1354         struct selector_tailq selectors;
1355         struct regarray_tailq regarrays;
1356         struct meter_profile_tailq meter_profiles;
1357         struct metarray_tailq metarrays;
1358
1359         struct port_in_runtime *in;
1360         struct port_out_runtime *out;
1361         struct instruction **action_instructions;
1362         struct rte_swx_table_state *table_state;
1363         struct table_statistics *table_stats;
1364         struct selector_statistics *selector_stats;
1365         struct regarray_runtime *regarray_runtime;
1366         struct metarray_runtime *metarray_runtime;
1367         struct instruction *instructions;
1368         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1369
1370         uint32_t n_structs;
1371         uint32_t n_ports_in;
1372         uint32_t n_ports_out;
1373         uint32_t n_extern_objs;
1374         uint32_t n_extern_funcs;
1375         uint32_t n_actions;
1376         uint32_t n_tables;
1377         uint32_t n_selectors;
1378         uint32_t n_regarrays;
1379         uint32_t n_metarrays;
1380         uint32_t n_headers;
1381         uint32_t thread_id;
1382         uint32_t port_id;
1383         uint32_t n_instructions;
1384         int build_done;
1385         int numa_node;
1386 };
1387
1388 /*
1389  * Struct.
1390  */
1391 static struct struct_type *
1392 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1393 {
1394         struct struct_type *elem;
1395
1396         TAILQ_FOREACH(elem, &p->struct_types, node)
1397                 if (strcmp(elem->name, name) == 0)
1398                         return elem;
1399
1400         return NULL;
1401 }
1402
1403 static struct field *
1404 struct_type_field_find(struct struct_type *st, const char *name)
1405 {
1406         uint32_t i;
1407
1408         for (i = 0; i < st->n_fields; i++) {
1409                 struct field *f = &st->fields[i];
1410
1411                 if (strcmp(f->name, name) == 0)
1412                         return f;
1413         }
1414
1415         return NULL;
1416 }
1417
1418 int
1419 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1420                                       const char *name,
1421                                       struct rte_swx_field_params *fields,
1422                                       uint32_t n_fields,
1423                                       int last_field_has_variable_size)
1424 {
1425         struct struct_type *st;
1426         uint32_t i;
1427
1428         CHECK(p, EINVAL);
1429         CHECK_NAME(name, EINVAL);
1430         CHECK(fields, EINVAL);
1431         CHECK(n_fields, EINVAL);
1432
1433         for (i = 0; i < n_fields; i++) {
1434                 struct rte_swx_field_params *f = &fields[i];
1435                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
1436                 uint32_t j;
1437
1438                 CHECK_NAME(f->name, EINVAL);
1439                 CHECK(f->n_bits, EINVAL);
1440                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
1441                 CHECK((f->n_bits & 7) == 0, EINVAL);
1442
1443                 for (j = 0; j < i; j++) {
1444                         struct rte_swx_field_params *f_prev = &fields[j];
1445
1446                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
1447                 }
1448         }
1449
1450         CHECK(!struct_type_find(p, name), EEXIST);
1451
1452         /* Node allocation. */
1453         st = calloc(1, sizeof(struct struct_type));
1454         CHECK(st, ENOMEM);
1455
1456         st->fields = calloc(n_fields, sizeof(struct field));
1457         if (!st->fields) {
1458                 free(st);
1459                 CHECK(0, ENOMEM);
1460         }
1461
1462         /* Node initialization. */
1463         strcpy(st->name, name);
1464         for (i = 0; i < n_fields; i++) {
1465                 struct field *dst = &st->fields[i];
1466                 struct rte_swx_field_params *src = &fields[i];
1467                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
1468
1469                 strcpy(dst->name, src->name);
1470                 dst->n_bits = src->n_bits;
1471                 dst->offset = st->n_bits;
1472                 dst->var_size = var_size;
1473
1474                 st->n_bits += src->n_bits;
1475                 st->n_bits_min += var_size ? 0 : src->n_bits;
1476         }
1477         st->n_fields = n_fields;
1478         st->var_size = last_field_has_variable_size;
1479
1480         /* Node add to tailq. */
1481         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1482
1483         return 0;
1484 }
1485
1486 static int
1487 struct_build(struct rte_swx_pipeline *p)
1488 {
1489         uint32_t i;
1490
1491         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1492                 struct thread *t = &p->threads[i];
1493
1494                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1495                 CHECK(t->structs, ENOMEM);
1496         }
1497
1498         return 0;
1499 }
1500
1501 static void
1502 struct_build_free(struct rte_swx_pipeline *p)
1503 {
1504         uint32_t i;
1505
1506         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1507                 struct thread *t = &p->threads[i];
1508
1509                 free(t->structs);
1510                 t->structs = NULL;
1511         }
1512 }
1513
1514 static void
1515 struct_free(struct rte_swx_pipeline *p)
1516 {
1517         struct_build_free(p);
1518
1519         /* Struct types. */
1520         for ( ; ; ) {
1521                 struct struct_type *elem;
1522
1523                 elem = TAILQ_FIRST(&p->struct_types);
1524                 if (!elem)
1525                         break;
1526
1527                 TAILQ_REMOVE(&p->struct_types, elem, node);
1528                 free(elem->fields);
1529                 free(elem);
1530         }
1531 }
1532
1533 /*
1534  * Input port.
1535  */
1536 static struct port_in_type *
1537 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1538 {
1539         struct port_in_type *elem;
1540
1541         if (!name)
1542                 return NULL;
1543
1544         TAILQ_FOREACH(elem, &p->port_in_types, node)
1545                 if (strcmp(elem->name, name) == 0)
1546                         return elem;
1547
1548         return NULL;
1549 }
1550
1551 int
1552 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1553                                        const char *name,
1554                                        struct rte_swx_port_in_ops *ops)
1555 {
1556         struct port_in_type *elem;
1557
1558         CHECK(p, EINVAL);
1559         CHECK_NAME(name, EINVAL);
1560         CHECK(ops, EINVAL);
1561         CHECK(ops->create, EINVAL);
1562         CHECK(ops->free, EINVAL);
1563         CHECK(ops->pkt_rx, EINVAL);
1564         CHECK(ops->stats_read, EINVAL);
1565
1566         CHECK(!port_in_type_find(p, name), EEXIST);
1567
1568         /* Node allocation. */
1569         elem = calloc(1, sizeof(struct port_in_type));
1570         CHECK(elem, ENOMEM);
1571
1572         /* Node initialization. */
1573         strcpy(elem->name, name);
1574         memcpy(&elem->ops, ops, sizeof(*ops));
1575
1576         /* Node add to tailq. */
1577         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1578
1579         return 0;
1580 }
1581
1582 static struct port_in *
1583 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1584 {
1585         struct port_in *port;
1586
1587         TAILQ_FOREACH(port, &p->ports_in, node)
1588                 if (port->id == port_id)
1589                         return port;
1590
1591         return NULL;
1592 }
1593
1594 int
1595 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1596                                 uint32_t port_id,
1597                                 const char *port_type_name,
1598                                 void *args)
1599 {
1600         struct port_in_type *type = NULL;
1601         struct port_in *port = NULL;
1602         void *obj = NULL;
1603
1604         CHECK(p, EINVAL);
1605
1606         CHECK(!port_in_find(p, port_id), EINVAL);
1607
1608         CHECK_NAME(port_type_name, EINVAL);
1609         type = port_in_type_find(p, port_type_name);
1610         CHECK(type, EINVAL);
1611
1612         obj = type->ops.create(args);
1613         CHECK(obj, ENODEV);
1614
1615         /* Node allocation. */
1616         port = calloc(1, sizeof(struct port_in));
1617         CHECK(port, ENOMEM);
1618
1619         /* Node initialization. */
1620         port->type = type;
1621         port->obj = obj;
1622         port->id = port_id;
1623
1624         /* Node add to tailq. */
1625         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1626         if (p->n_ports_in < port_id + 1)
1627                 p->n_ports_in = port_id + 1;
1628
1629         return 0;
1630 }
1631
1632 static int
1633 port_in_build(struct rte_swx_pipeline *p)
1634 {
1635         struct port_in *port;
1636         uint32_t i;
1637
1638         CHECK(p->n_ports_in, EINVAL);
1639         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1640
1641         for (i = 0; i < p->n_ports_in; i++)
1642                 CHECK(port_in_find(p, i), EINVAL);
1643
1644         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1645         CHECK(p->in, ENOMEM);
1646
1647         TAILQ_FOREACH(port, &p->ports_in, node) {
1648                 struct port_in_runtime *in = &p->in[port->id];
1649
1650                 in->pkt_rx = port->type->ops.pkt_rx;
1651                 in->obj = port->obj;
1652         }
1653
1654         return 0;
1655 }
1656
1657 static void
1658 port_in_build_free(struct rte_swx_pipeline *p)
1659 {
1660         free(p->in);
1661         p->in = NULL;
1662 }
1663
1664 static void
1665 port_in_free(struct rte_swx_pipeline *p)
1666 {
1667         port_in_build_free(p);
1668
1669         /* Input ports. */
1670         for ( ; ; ) {
1671                 struct port_in *port;
1672
1673                 port = TAILQ_FIRST(&p->ports_in);
1674                 if (!port)
1675                         break;
1676
1677                 TAILQ_REMOVE(&p->ports_in, port, node);
1678                 port->type->ops.free(port->obj);
1679                 free(port);
1680         }
1681
1682         /* Input port types. */
1683         for ( ; ; ) {
1684                 struct port_in_type *elem;
1685
1686                 elem = TAILQ_FIRST(&p->port_in_types);
1687                 if (!elem)
1688                         break;
1689
1690                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1691                 free(elem);
1692         }
1693 }
1694
1695 /*
1696  * Output port.
1697  */
1698 static struct port_out_type *
1699 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1700 {
1701         struct port_out_type *elem;
1702
1703         if (!name)
1704                 return NULL;
1705
1706         TAILQ_FOREACH(elem, &p->port_out_types, node)
1707                 if (!strcmp(elem->name, name))
1708                         return elem;
1709
1710         return NULL;
1711 }
1712
1713 int
1714 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1715                                         const char *name,
1716                                         struct rte_swx_port_out_ops *ops)
1717 {
1718         struct port_out_type *elem;
1719
1720         CHECK(p, EINVAL);
1721         CHECK_NAME(name, EINVAL);
1722         CHECK(ops, EINVAL);
1723         CHECK(ops->create, EINVAL);
1724         CHECK(ops->free, EINVAL);
1725         CHECK(ops->pkt_tx, EINVAL);
1726         CHECK(ops->stats_read, EINVAL);
1727
1728         CHECK(!port_out_type_find(p, name), EEXIST);
1729
1730         /* Node allocation. */
1731         elem = calloc(1, sizeof(struct port_out_type));
1732         CHECK(elem, ENOMEM);
1733
1734         /* Node initialization. */
1735         strcpy(elem->name, name);
1736         memcpy(&elem->ops, ops, sizeof(*ops));
1737
1738         /* Node add to tailq. */
1739         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1740
1741         return 0;
1742 }
1743
1744 static struct port_out *
1745 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1746 {
1747         struct port_out *port;
1748
1749         TAILQ_FOREACH(port, &p->ports_out, node)
1750                 if (port->id == port_id)
1751                         return port;
1752
1753         return NULL;
1754 }
1755
1756 int
1757 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1758                                  uint32_t port_id,
1759                                  const char *port_type_name,
1760                                  void *args)
1761 {
1762         struct port_out_type *type = NULL;
1763         struct port_out *port = NULL;
1764         void *obj = NULL;
1765
1766         CHECK(p, EINVAL);
1767
1768         CHECK(!port_out_find(p, port_id), EINVAL);
1769
1770         CHECK_NAME(port_type_name, EINVAL);
1771         type = port_out_type_find(p, port_type_name);
1772         CHECK(type, EINVAL);
1773
1774         obj = type->ops.create(args);
1775         CHECK(obj, ENODEV);
1776
1777         /* Node allocation. */
1778         port = calloc(1, sizeof(struct port_out));
1779         CHECK(port, ENOMEM);
1780
1781         /* Node initialization. */
1782         port->type = type;
1783         port->obj = obj;
1784         port->id = port_id;
1785
1786         /* Node add to tailq. */
1787         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1788         if (p->n_ports_out < port_id + 1)
1789                 p->n_ports_out = port_id + 1;
1790
1791         return 0;
1792 }
1793
1794 static int
1795 port_out_build(struct rte_swx_pipeline *p)
1796 {
1797         struct port_out *port;
1798         uint32_t i;
1799
1800         CHECK(p->n_ports_out, EINVAL);
1801
1802         for (i = 0; i < p->n_ports_out; i++)
1803                 CHECK(port_out_find(p, i), EINVAL);
1804
1805         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1806         CHECK(p->out, ENOMEM);
1807
1808         TAILQ_FOREACH(port, &p->ports_out, node) {
1809                 struct port_out_runtime *out = &p->out[port->id];
1810
1811                 out->pkt_tx = port->type->ops.pkt_tx;
1812                 out->flush = port->type->ops.flush;
1813                 out->obj = port->obj;
1814         }
1815
1816         return 0;
1817 }
1818
1819 static void
1820 port_out_build_free(struct rte_swx_pipeline *p)
1821 {
1822         free(p->out);
1823         p->out = NULL;
1824 }
1825
1826 static void
1827 port_out_free(struct rte_swx_pipeline *p)
1828 {
1829         port_out_build_free(p);
1830
1831         /* Output ports. */
1832         for ( ; ; ) {
1833                 struct port_out *port;
1834
1835                 port = TAILQ_FIRST(&p->ports_out);
1836                 if (!port)
1837                         break;
1838
1839                 TAILQ_REMOVE(&p->ports_out, port, node);
1840                 port->type->ops.free(port->obj);
1841                 free(port);
1842         }
1843
1844         /* Output port types. */
1845         for ( ; ; ) {
1846                 struct port_out_type *elem;
1847
1848                 elem = TAILQ_FIRST(&p->port_out_types);
1849                 if (!elem)
1850                         break;
1851
1852                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1853                 free(elem);
1854         }
1855 }
1856
1857 /*
1858  * Extern object.
1859  */
1860 static struct extern_type *
1861 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1862 {
1863         struct extern_type *elem;
1864
1865         TAILQ_FOREACH(elem, &p->extern_types, node)
1866                 if (strcmp(elem->name, name) == 0)
1867                         return elem;
1868
1869         return NULL;
1870 }
1871
1872 static struct extern_type_member_func *
1873 extern_type_member_func_find(struct extern_type *type, const char *name)
1874 {
1875         struct extern_type_member_func *elem;
1876
1877         TAILQ_FOREACH(elem, &type->funcs, node)
1878                 if (strcmp(elem->name, name) == 0)
1879                         return elem;
1880
1881         return NULL;
1882 }
1883
1884 static struct extern_obj *
1885 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1886 {
1887         struct extern_obj *elem;
1888
1889         TAILQ_FOREACH(elem, &p->extern_objs, node)
1890                 if (strcmp(elem->name, name) == 0)
1891                         return elem;
1892
1893         return NULL;
1894 }
1895
1896 static struct extern_type_member_func *
1897 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1898                              const char *name,
1899                              struct extern_obj **obj)
1900 {
1901         struct extern_obj *object;
1902         struct extern_type_member_func *func;
1903         char *object_name, *func_name;
1904
1905         if (name[0] != 'e' || name[1] != '.')
1906                 return NULL;
1907
1908         object_name = strdup(&name[2]);
1909         if (!object_name)
1910                 return NULL;
1911
1912         func_name = strchr(object_name, '.');
1913         if (!func_name) {
1914                 free(object_name);
1915                 return NULL;
1916         }
1917
1918         *func_name = 0;
1919         func_name++;
1920
1921         object = extern_obj_find(p, object_name);
1922         if (!object) {
1923                 free(object_name);
1924                 return NULL;
1925         }
1926
1927         func = extern_type_member_func_find(object->type, func_name);
1928         if (!func) {
1929                 free(object_name);
1930                 return NULL;
1931         }
1932
1933         if (obj)
1934                 *obj = object;
1935
1936         free(object_name);
1937         return func;
1938 }
1939
1940 static struct field *
1941 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1942                                const char *name,
1943                                struct extern_obj **object)
1944 {
1945         struct extern_obj *obj;
1946         struct field *f;
1947         char *obj_name, *field_name;
1948
1949         if ((name[0] != 'e') || (name[1] != '.'))
1950                 return NULL;
1951
1952         obj_name = strdup(&name[2]);
1953         if (!obj_name)
1954                 return NULL;
1955
1956         field_name = strchr(obj_name, '.');
1957         if (!field_name) {
1958                 free(obj_name);
1959                 return NULL;
1960         }
1961
1962         *field_name = 0;
1963         field_name++;
1964
1965         obj = extern_obj_find(p, obj_name);
1966         if (!obj) {
1967                 free(obj_name);
1968                 return NULL;
1969         }
1970
1971         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1972         if (!f) {
1973                 free(obj_name);
1974                 return NULL;
1975         }
1976
1977         if (object)
1978                 *object = obj;
1979
1980         free(obj_name);
1981         return f;
1982 }
1983
1984 int
1985 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1986         const char *name,
1987         const char *mailbox_struct_type_name,
1988         rte_swx_extern_type_constructor_t constructor,
1989         rte_swx_extern_type_destructor_t destructor)
1990 {
1991         struct extern_type *elem;
1992         struct struct_type *mailbox_struct_type;
1993
1994         CHECK(p, EINVAL);
1995
1996         CHECK_NAME(name, EINVAL);
1997         CHECK(!extern_type_find(p, name), EEXIST);
1998
1999         CHECK_NAME(mailbox_struct_type_name, EINVAL);
2000         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2001         CHECK(mailbox_struct_type, EINVAL);
2002         CHECK(!mailbox_struct_type->var_size, EINVAL);
2003
2004         CHECK(constructor, EINVAL);
2005         CHECK(destructor, EINVAL);
2006
2007         /* Node allocation. */
2008         elem = calloc(1, sizeof(struct extern_type));
2009         CHECK(elem, ENOMEM);
2010
2011         /* Node initialization. */
2012         strcpy(elem->name, name);
2013         elem->mailbox_struct_type = mailbox_struct_type;
2014         elem->constructor = constructor;
2015         elem->destructor = destructor;
2016         TAILQ_INIT(&elem->funcs);
2017
2018         /* Node add to tailq. */
2019         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
2020
2021         return 0;
2022 }
2023
2024 int
2025 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
2026         const char *extern_type_name,
2027         const char *name,
2028         rte_swx_extern_type_member_func_t member_func)
2029 {
2030         struct extern_type *type;
2031         struct extern_type_member_func *type_member;
2032
2033         CHECK(p, EINVAL);
2034
2035         CHECK_NAME(extern_type_name, EINVAL);
2036         type = extern_type_find(p, extern_type_name);
2037         CHECK(type, EINVAL);
2038         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
2039
2040         CHECK_NAME(name, EINVAL);
2041         CHECK(!extern_type_member_func_find(type, name), EEXIST);
2042
2043         CHECK(member_func, EINVAL);
2044
2045         /* Node allocation. */
2046         type_member = calloc(1, sizeof(struct extern_type_member_func));
2047         CHECK(type_member, ENOMEM);
2048
2049         /* Node initialization. */
2050         strcpy(type_member->name, name);
2051         type_member->func = member_func;
2052         type_member->id = type->n_funcs;
2053
2054         /* Node add to tailq. */
2055         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
2056         type->n_funcs++;
2057
2058         return 0;
2059 }
2060
2061 int
2062 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
2063                                       const char *extern_type_name,
2064                                       const char *name,
2065                                       const char *args)
2066 {
2067         struct extern_type *type;
2068         struct extern_obj *obj;
2069         void *obj_handle;
2070
2071         CHECK(p, EINVAL);
2072
2073         CHECK_NAME(extern_type_name, EINVAL);
2074         type = extern_type_find(p, extern_type_name);
2075         CHECK(type, EINVAL);
2076
2077         CHECK_NAME(name, EINVAL);
2078         CHECK(!extern_obj_find(p, name), EEXIST);
2079
2080         /* Node allocation. */
2081         obj = calloc(1, sizeof(struct extern_obj));
2082         CHECK(obj, ENOMEM);
2083
2084         /* Object construction. */
2085         obj_handle = type->constructor(args);
2086         if (!obj_handle) {
2087                 free(obj);
2088                 CHECK(0, ENODEV);
2089         }
2090
2091         /* Node initialization. */
2092         strcpy(obj->name, name);
2093         obj->type = type;
2094         obj->obj = obj_handle;
2095         obj->struct_id = p->n_structs;
2096         obj->id = p->n_extern_objs;
2097
2098         /* Node add to tailq. */
2099         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
2100         p->n_extern_objs++;
2101         p->n_structs++;
2102
2103         return 0;
2104 }
2105
2106 static int
2107 extern_obj_build(struct rte_swx_pipeline *p)
2108 {
2109         uint32_t i;
2110
2111         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2112                 struct thread *t = &p->threads[i];
2113                 struct extern_obj *obj;
2114
2115                 t->extern_objs = calloc(p->n_extern_objs,
2116                                         sizeof(struct extern_obj_runtime));
2117                 CHECK(t->extern_objs, ENOMEM);
2118
2119                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
2120                         struct extern_obj_runtime *r =
2121                                 &t->extern_objs[obj->id];
2122                         struct extern_type_member_func *func;
2123                         uint32_t mailbox_size =
2124                                 obj->type->mailbox_struct_type->n_bits / 8;
2125
2126                         r->obj = obj->obj;
2127
2128                         r->mailbox = calloc(1, mailbox_size);
2129                         CHECK(r->mailbox, ENOMEM);
2130
2131                         TAILQ_FOREACH(func, &obj->type->funcs, node)
2132                                 r->funcs[func->id] = func->func;
2133
2134                         t->structs[obj->struct_id] = r->mailbox;
2135                 }
2136         }
2137
2138         return 0;
2139 }
2140
2141 static void
2142 extern_obj_build_free(struct rte_swx_pipeline *p)
2143 {
2144         uint32_t i;
2145
2146         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2147                 struct thread *t = &p->threads[i];
2148                 uint32_t j;
2149
2150                 if (!t->extern_objs)
2151                         continue;
2152
2153                 for (j = 0; j < p->n_extern_objs; j++) {
2154                         struct extern_obj_runtime *r = &t->extern_objs[j];
2155
2156                         free(r->mailbox);
2157                 }
2158
2159                 free(t->extern_objs);
2160                 t->extern_objs = NULL;
2161         }
2162 }
2163
2164 static void
2165 extern_obj_free(struct rte_swx_pipeline *p)
2166 {
2167         extern_obj_build_free(p);
2168
2169         /* Extern objects. */
2170         for ( ; ; ) {
2171                 struct extern_obj *elem;
2172
2173                 elem = TAILQ_FIRST(&p->extern_objs);
2174                 if (!elem)
2175                         break;
2176
2177                 TAILQ_REMOVE(&p->extern_objs, elem, node);
2178                 if (elem->obj)
2179                         elem->type->destructor(elem->obj);
2180                 free(elem);
2181         }
2182
2183         /* Extern types. */
2184         for ( ; ; ) {
2185                 struct extern_type *elem;
2186
2187                 elem = TAILQ_FIRST(&p->extern_types);
2188                 if (!elem)
2189                         break;
2190
2191                 TAILQ_REMOVE(&p->extern_types, elem, node);
2192
2193                 for ( ; ; ) {
2194                         struct extern_type_member_func *func;
2195
2196                         func = TAILQ_FIRST(&elem->funcs);
2197                         if (!func)
2198                                 break;
2199
2200                         TAILQ_REMOVE(&elem->funcs, func, node);
2201                         free(func);
2202                 }
2203
2204                 free(elem);
2205         }
2206 }
2207
2208 /*
2209  * Extern function.
2210  */
2211 static struct extern_func *
2212 extern_func_find(struct rte_swx_pipeline *p, const char *name)
2213 {
2214         struct extern_func *elem;
2215
2216         TAILQ_FOREACH(elem, &p->extern_funcs, node)
2217                 if (strcmp(elem->name, name) == 0)
2218                         return elem;
2219
2220         return NULL;
2221 }
2222
2223 static struct extern_func *
2224 extern_func_parse(struct rte_swx_pipeline *p,
2225                   const char *name)
2226 {
2227         if (name[0] != 'f' || name[1] != '.')
2228                 return NULL;
2229
2230         return extern_func_find(p, &name[2]);
2231 }
2232
2233 static struct field *
2234 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
2235                                 const char *name,
2236                                 struct extern_func **function)
2237 {
2238         struct extern_func *func;
2239         struct field *f;
2240         char *func_name, *field_name;
2241
2242         if ((name[0] != 'f') || (name[1] != '.'))
2243                 return NULL;
2244
2245         func_name = strdup(&name[2]);
2246         if (!func_name)
2247                 return NULL;
2248
2249         field_name = strchr(func_name, '.');
2250         if (!field_name) {
2251                 free(func_name);
2252                 return NULL;
2253         }
2254
2255         *field_name = 0;
2256         field_name++;
2257
2258         func = extern_func_find(p, func_name);
2259         if (!func) {
2260                 free(func_name);
2261                 return NULL;
2262         }
2263
2264         f = struct_type_field_find(func->mailbox_struct_type, field_name);
2265         if (!f) {
2266                 free(func_name);
2267                 return NULL;
2268         }
2269
2270         if (function)
2271                 *function = func;
2272
2273         free(func_name);
2274         return f;
2275 }
2276
2277 int
2278 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
2279                                       const char *name,
2280                                       const char *mailbox_struct_type_name,
2281                                       rte_swx_extern_func_t func)
2282 {
2283         struct extern_func *f;
2284         struct struct_type *mailbox_struct_type;
2285
2286         CHECK(p, EINVAL);
2287
2288         CHECK_NAME(name, EINVAL);
2289         CHECK(!extern_func_find(p, name), EEXIST);
2290
2291         CHECK_NAME(mailbox_struct_type_name, EINVAL);
2292         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2293         CHECK(mailbox_struct_type, EINVAL);
2294         CHECK(!mailbox_struct_type->var_size, EINVAL);
2295
2296         CHECK(func, EINVAL);
2297
2298         /* Node allocation. */
2299         f = calloc(1, sizeof(struct extern_func));
2300         CHECK(func, ENOMEM);
2301
2302         /* Node initialization. */
2303         strcpy(f->name, name);
2304         f->mailbox_struct_type = mailbox_struct_type;
2305         f->func = func;
2306         f->struct_id = p->n_structs;
2307         f->id = p->n_extern_funcs;
2308
2309         /* Node add to tailq. */
2310         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
2311         p->n_extern_funcs++;
2312         p->n_structs++;
2313
2314         return 0;
2315 }
2316
2317 static int
2318 extern_func_build(struct rte_swx_pipeline *p)
2319 {
2320         uint32_t i;
2321
2322         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2323                 struct thread *t = &p->threads[i];
2324                 struct extern_func *func;
2325
2326                 /* Memory allocation. */
2327                 t->extern_funcs = calloc(p->n_extern_funcs,
2328                                          sizeof(struct extern_func_runtime));
2329                 CHECK(t->extern_funcs, ENOMEM);
2330
2331                 /* Extern function. */
2332                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
2333                         struct extern_func_runtime *r =
2334                                 &t->extern_funcs[func->id];
2335                         uint32_t mailbox_size =
2336                                 func->mailbox_struct_type->n_bits / 8;
2337
2338                         r->func = func->func;
2339
2340                         r->mailbox = calloc(1, mailbox_size);
2341                         CHECK(r->mailbox, ENOMEM);
2342
2343                         t->structs[func->struct_id] = r->mailbox;
2344                 }
2345         }
2346
2347         return 0;
2348 }
2349
2350 static void
2351 extern_func_build_free(struct rte_swx_pipeline *p)
2352 {
2353         uint32_t i;
2354
2355         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2356                 struct thread *t = &p->threads[i];
2357                 uint32_t j;
2358
2359                 if (!t->extern_funcs)
2360                         continue;
2361
2362                 for (j = 0; j < p->n_extern_funcs; j++) {
2363                         struct extern_func_runtime *r = &t->extern_funcs[j];
2364
2365                         free(r->mailbox);
2366                 }
2367
2368                 free(t->extern_funcs);
2369                 t->extern_funcs = NULL;
2370         }
2371 }
2372
2373 static void
2374 extern_func_free(struct rte_swx_pipeline *p)
2375 {
2376         extern_func_build_free(p);
2377
2378         for ( ; ; ) {
2379                 struct extern_func *elem;
2380
2381                 elem = TAILQ_FIRST(&p->extern_funcs);
2382                 if (!elem)
2383                         break;
2384
2385                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
2386                 free(elem);
2387         }
2388 }
2389
2390 /*
2391  * Header.
2392  */
2393 static struct header *
2394 header_find(struct rte_swx_pipeline *p, const char *name)
2395 {
2396         struct header *elem;
2397
2398         TAILQ_FOREACH(elem, &p->headers, node)
2399                 if (strcmp(elem->name, name) == 0)
2400                         return elem;
2401
2402         return NULL;
2403 }
2404
2405 static struct header *
2406 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
2407 {
2408         struct header *elem;
2409
2410         TAILQ_FOREACH(elem, &p->headers, node)
2411                 if (elem->struct_id == struct_id)
2412                         return elem;
2413
2414         return NULL;
2415 }
2416
2417 static struct header *
2418 header_parse(struct rte_swx_pipeline *p,
2419              const char *name)
2420 {
2421         if (name[0] != 'h' || name[1] != '.')
2422                 return NULL;
2423
2424         return header_find(p, &name[2]);
2425 }
2426
2427 static struct field *
2428 header_field_parse(struct rte_swx_pipeline *p,
2429                    const char *name,
2430                    struct header **header)
2431 {
2432         struct header *h;
2433         struct field *f;
2434         char *header_name, *field_name;
2435
2436         if ((name[0] != 'h') || (name[1] != '.'))
2437                 return NULL;
2438
2439         header_name = strdup(&name[2]);
2440         if (!header_name)
2441                 return NULL;
2442
2443         field_name = strchr(header_name, '.');
2444         if (!field_name) {
2445                 free(header_name);
2446                 return NULL;
2447         }
2448
2449         *field_name = 0;
2450         field_name++;
2451
2452         h = header_find(p, header_name);
2453         if (!h) {
2454                 free(header_name);
2455                 return NULL;
2456         }
2457
2458         f = struct_type_field_find(h->st, field_name);
2459         if (!f) {
2460                 free(header_name);
2461                 return NULL;
2462         }
2463
2464         if (header)
2465                 *header = h;
2466
2467         free(header_name);
2468         return f;
2469 }
2470
2471 int
2472 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2473                                         const char *name,
2474                                         const char *struct_type_name)
2475 {
2476         struct struct_type *st;
2477         struct header *h;
2478         size_t n_headers_max;
2479
2480         CHECK(p, EINVAL);
2481         CHECK_NAME(name, EINVAL);
2482         CHECK_NAME(struct_type_name, EINVAL);
2483
2484         CHECK(!header_find(p, name), EEXIST);
2485
2486         st = struct_type_find(p, struct_type_name);
2487         CHECK(st, EINVAL);
2488
2489         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2490         CHECK(p->n_headers < n_headers_max, ENOSPC);
2491
2492         /* Node allocation. */
2493         h = calloc(1, sizeof(struct header));
2494         CHECK(h, ENOMEM);
2495
2496         /* Node initialization. */
2497         strcpy(h->name, name);
2498         h->st = st;
2499         h->struct_id = p->n_structs;
2500         h->id = p->n_headers;
2501
2502         /* Node add to tailq. */
2503         TAILQ_INSERT_TAIL(&p->headers, h, node);
2504         p->n_headers++;
2505         p->n_structs++;
2506
2507         return 0;
2508 }
2509
2510 static int
2511 header_build(struct rte_swx_pipeline *p)
2512 {
2513         struct header *h;
2514         uint32_t n_bytes = 0, i;
2515
2516         TAILQ_FOREACH(h, &p->headers, node) {
2517                 n_bytes += h->st->n_bits / 8;
2518         }
2519
2520         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2521                 struct thread *t = &p->threads[i];
2522                 uint32_t offset = 0;
2523
2524                 t->headers = calloc(p->n_headers,
2525                                     sizeof(struct header_runtime));
2526                 CHECK(t->headers, ENOMEM);
2527
2528                 t->headers_out = calloc(p->n_headers,
2529                                         sizeof(struct header_out_runtime));
2530                 CHECK(t->headers_out, ENOMEM);
2531
2532                 t->header_storage = calloc(1, n_bytes);
2533                 CHECK(t->header_storage, ENOMEM);
2534
2535                 t->header_out_storage = calloc(1, n_bytes);
2536                 CHECK(t->header_out_storage, ENOMEM);
2537
2538                 TAILQ_FOREACH(h, &p->headers, node) {
2539                         uint8_t *header_storage;
2540                         uint32_t n_bytes =  h->st->n_bits / 8;
2541
2542                         header_storage = &t->header_storage[offset];
2543                         offset += n_bytes;
2544
2545                         t->headers[h->id].ptr0 = header_storage;
2546                         t->headers[h->id].n_bytes = n_bytes;
2547
2548                         t->structs[h->struct_id] = header_storage;
2549                 }
2550         }
2551
2552         return 0;
2553 }
2554
2555 static void
2556 header_build_free(struct rte_swx_pipeline *p)
2557 {
2558         uint32_t i;
2559
2560         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2561                 struct thread *t = &p->threads[i];
2562
2563                 free(t->headers_out);
2564                 t->headers_out = NULL;
2565
2566                 free(t->headers);
2567                 t->headers = NULL;
2568
2569                 free(t->header_out_storage);
2570                 t->header_out_storage = NULL;
2571
2572                 free(t->header_storage);
2573                 t->header_storage = NULL;
2574         }
2575 }
2576
2577 static void
2578 header_free(struct rte_swx_pipeline *p)
2579 {
2580         header_build_free(p);
2581
2582         for ( ; ; ) {
2583                 struct header *elem;
2584
2585                 elem = TAILQ_FIRST(&p->headers);
2586                 if (!elem)
2587                         break;
2588
2589                 TAILQ_REMOVE(&p->headers, elem, node);
2590                 free(elem);
2591         }
2592 }
2593
2594 /*
2595  * Meta-data.
2596  */
2597 static struct field *
2598 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2599 {
2600         if (!p->metadata_st)
2601                 return NULL;
2602
2603         if (name[0] != 'm' || name[1] != '.')
2604                 return NULL;
2605
2606         return struct_type_field_find(p->metadata_st, &name[2]);
2607 }
2608
2609 int
2610 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2611                                           const char *struct_type_name)
2612 {
2613         struct struct_type *st = NULL;
2614
2615         CHECK(p, EINVAL);
2616
2617         CHECK_NAME(struct_type_name, EINVAL);
2618         st  = struct_type_find(p, struct_type_name);
2619         CHECK(st, EINVAL);
2620         CHECK(!st->var_size, EINVAL);
2621         CHECK(!p->metadata_st, EINVAL);
2622
2623         p->metadata_st = st;
2624         p->metadata_struct_id = p->n_structs;
2625
2626         p->n_structs++;
2627
2628         return 0;
2629 }
2630
2631 static int
2632 metadata_build(struct rte_swx_pipeline *p)
2633 {
2634         uint32_t n_bytes = p->metadata_st->n_bits / 8;
2635         uint32_t i;
2636
2637         /* Thread-level initialization. */
2638         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2639                 struct thread *t = &p->threads[i];
2640                 uint8_t *metadata;
2641
2642                 metadata = calloc(1, n_bytes);
2643                 CHECK(metadata, ENOMEM);
2644
2645                 t->metadata = metadata;
2646                 t->structs[p->metadata_struct_id] = metadata;
2647         }
2648
2649         return 0;
2650 }
2651
2652 static void
2653 metadata_build_free(struct rte_swx_pipeline *p)
2654 {
2655         uint32_t i;
2656
2657         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2658                 struct thread *t = &p->threads[i];
2659
2660                 free(t->metadata);
2661                 t->metadata = NULL;
2662         }
2663 }
2664
2665 static void
2666 metadata_free(struct rte_swx_pipeline *p)
2667 {
2668         metadata_build_free(p);
2669 }
2670
2671 /*
2672  * Instruction.
2673  */
2674 static int
2675 instruction_is_tx(enum instruction_type type)
2676 {
2677         switch (type) {
2678         case INSTR_TX:
2679         case INSTR_TX_I:
2680                 return 1;
2681
2682         default:
2683                 return 0;
2684         }
2685 }
2686
2687 static int
2688 instruction_is_jmp(struct instruction *instr)
2689 {
2690         switch (instr->type) {
2691         case INSTR_JMP:
2692         case INSTR_JMP_VALID:
2693         case INSTR_JMP_INVALID:
2694         case INSTR_JMP_HIT:
2695         case INSTR_JMP_MISS:
2696         case INSTR_JMP_ACTION_HIT:
2697         case INSTR_JMP_ACTION_MISS:
2698         case INSTR_JMP_EQ:
2699         case INSTR_JMP_EQ_MH:
2700         case INSTR_JMP_EQ_HM:
2701         case INSTR_JMP_EQ_HH:
2702         case INSTR_JMP_EQ_I:
2703         case INSTR_JMP_NEQ:
2704         case INSTR_JMP_NEQ_MH:
2705         case INSTR_JMP_NEQ_HM:
2706         case INSTR_JMP_NEQ_HH:
2707         case INSTR_JMP_NEQ_I:
2708         case INSTR_JMP_LT:
2709         case INSTR_JMP_LT_MH:
2710         case INSTR_JMP_LT_HM:
2711         case INSTR_JMP_LT_HH:
2712         case INSTR_JMP_LT_MI:
2713         case INSTR_JMP_LT_HI:
2714         case INSTR_JMP_GT:
2715         case INSTR_JMP_GT_MH:
2716         case INSTR_JMP_GT_HM:
2717         case INSTR_JMP_GT_HH:
2718         case INSTR_JMP_GT_MI:
2719         case INSTR_JMP_GT_HI:
2720                 return 1;
2721
2722         default:
2723                 return 0;
2724         }
2725 }
2726
2727 static struct field *
2728 action_field_parse(struct action *action, const char *name);
2729
2730 static struct field *
2731 struct_field_parse(struct rte_swx_pipeline *p,
2732                    struct action *action,
2733                    const char *name,
2734                    uint32_t *struct_id)
2735 {
2736         struct field *f;
2737
2738         switch (name[0]) {
2739         case 'h':
2740         {
2741                 struct header *header;
2742
2743                 f = header_field_parse(p, name, &header);
2744                 if (!f)
2745                         return NULL;
2746
2747                 *struct_id = header->struct_id;
2748                 return f;
2749         }
2750
2751         case 'm':
2752         {
2753                 f = metadata_field_parse(p, name);
2754                 if (!f)
2755                         return NULL;
2756
2757                 *struct_id = p->metadata_struct_id;
2758                 return f;
2759         }
2760
2761         case 't':
2762         {
2763                 if (!action)
2764                         return NULL;
2765
2766                 f = action_field_parse(action, name);
2767                 if (!f)
2768                         return NULL;
2769
2770                 *struct_id = 0;
2771                 return f;
2772         }
2773
2774         case 'e':
2775         {
2776                 struct extern_obj *obj;
2777
2778                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2779                 if (!f)
2780                         return NULL;
2781
2782                 *struct_id = obj->struct_id;
2783                 return f;
2784         }
2785
2786         case 'f':
2787         {
2788                 struct extern_func *func;
2789
2790                 f = extern_func_mailbox_field_parse(p, name, &func);
2791                 if (!f)
2792                         return NULL;
2793
2794                 *struct_id = func->struct_id;
2795                 return f;
2796         }
2797
2798         default:
2799                 return NULL;
2800         }
2801 }
2802
2803 static inline void
2804 pipeline_port_inc(struct rte_swx_pipeline *p)
2805 {
2806         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2807 }
2808
2809 static inline void
2810 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2811 {
2812         t->ip = p->instructions;
2813 }
2814
2815 static inline void
2816 thread_ip_set(struct thread *t, struct instruction *ip)
2817 {
2818         t->ip = ip;
2819 }
2820
2821 static inline void
2822 thread_ip_action_call(struct rte_swx_pipeline *p,
2823                       struct thread *t,
2824                       uint32_t action_id)
2825 {
2826         t->ret = t->ip + 1;
2827         t->ip = p->action_instructions[action_id];
2828 }
2829
2830 static inline void
2831 thread_ip_inc(struct rte_swx_pipeline *p);
2832
2833 static inline void
2834 thread_ip_inc(struct rte_swx_pipeline *p)
2835 {
2836         struct thread *t = &p->threads[p->thread_id];
2837
2838         t->ip++;
2839 }
2840
2841 static inline void
2842 thread_ip_inc_cond(struct thread *t, int cond)
2843 {
2844         t->ip += cond;
2845 }
2846
2847 static inline void
2848 thread_yield(struct rte_swx_pipeline *p)
2849 {
2850         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2851 }
2852
2853 static inline void
2854 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2855 {
2856         p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2857 }
2858
2859 /*
2860  * rx.
2861  */
2862 static int
2863 instr_rx_translate(struct rte_swx_pipeline *p,
2864                    struct action *action,
2865                    char **tokens,
2866                    int n_tokens,
2867                    struct instruction *instr,
2868                    struct instruction_data *data __rte_unused)
2869 {
2870         struct field *f;
2871
2872         CHECK(!action, EINVAL);
2873         CHECK(n_tokens == 2, EINVAL);
2874
2875         f = metadata_field_parse(p, tokens[1]);
2876         CHECK(f, EINVAL);
2877
2878         instr->type = INSTR_RX;
2879         instr->io.io.offset = f->offset / 8;
2880         instr->io.io.n_bits = f->n_bits;
2881         return 0;
2882 }
2883
2884 static inline void
2885 instr_rx_exec(struct rte_swx_pipeline *p);
2886
2887 static inline void
2888 instr_rx_exec(struct rte_swx_pipeline *p)
2889 {
2890         struct thread *t = &p->threads[p->thread_id];
2891         struct instruction *ip = t->ip;
2892         struct port_in_runtime *port = &p->in[p->port_id];
2893         struct rte_swx_pkt *pkt = &t->pkt;
2894         int pkt_received;
2895
2896         /* Packet. */
2897         pkt_received = port->pkt_rx(port->obj, pkt);
2898         t->ptr = &pkt->pkt[pkt->offset];
2899         rte_prefetch0(t->ptr);
2900
2901         TRACE("[Thread %2u] rx %s from port %u\n",
2902               p->thread_id,
2903               pkt_received ? "1 pkt" : "0 pkts",
2904               p->port_id);
2905
2906         /* Headers. */
2907         t->valid_headers = 0;
2908         t->n_headers_out = 0;
2909
2910         /* Meta-data. */
2911         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2912
2913         /* Tables. */
2914         t->table_state = p->table_state;
2915
2916         /* Thread. */
2917         pipeline_port_inc(p);
2918         thread_ip_inc_cond(t, pkt_received);
2919         thread_yield(p);
2920 }
2921
2922 /*
2923  * tx.
2924  */
2925 static int
2926 instr_tx_translate(struct rte_swx_pipeline *p,
2927                    struct action *action __rte_unused,
2928                    char **tokens,
2929                    int n_tokens,
2930                    struct instruction *instr,
2931                    struct instruction_data *data __rte_unused)
2932 {
2933         char *port = tokens[1];
2934         struct field *f;
2935         uint32_t port_val;
2936
2937         CHECK(n_tokens == 2, EINVAL);
2938
2939         f = metadata_field_parse(p, port);
2940         if (f) {
2941                 instr->type = INSTR_TX;
2942                 instr->io.io.offset = f->offset / 8;
2943                 instr->io.io.n_bits = f->n_bits;
2944                 return 0;
2945         }
2946
2947         /* TX_I. */
2948         port_val = strtoul(port, &port, 0);
2949         CHECK(!port[0], EINVAL);
2950
2951         instr->type = INSTR_TX_I;
2952         instr->io.io.val = port_val;
2953         return 0;
2954 }
2955
2956 static int
2957 instr_drop_translate(struct rte_swx_pipeline *p,
2958                      struct action *action __rte_unused,
2959                      char **tokens __rte_unused,
2960                      int n_tokens,
2961                      struct instruction *instr,
2962                      struct instruction_data *data __rte_unused)
2963 {
2964         CHECK(n_tokens == 1, EINVAL);
2965
2966         /* TX_I. */
2967         instr->type = INSTR_TX_I;
2968         instr->io.io.val = p->n_ports_out - 1;
2969         return 0;
2970 }
2971
2972 static inline void
2973 emit_handler(struct thread *t)
2974 {
2975         struct header_out_runtime *h0 = &t->headers_out[0];
2976         struct header_out_runtime *h1 = &t->headers_out[1];
2977         uint32_t offset = 0, i;
2978
2979         /* No header change or header decapsulation. */
2980         if ((t->n_headers_out == 1) &&
2981             (h0->ptr + h0->n_bytes == t->ptr)) {
2982                 TRACE("Emit handler: no header change or header decap.\n");
2983
2984                 t->pkt.offset -= h0->n_bytes;
2985                 t->pkt.length += h0->n_bytes;
2986
2987                 return;
2988         }
2989
2990         /* Header encapsulation (optionally, with prior header decasulation). */
2991         if ((t->n_headers_out == 2) &&
2992             (h1->ptr + h1->n_bytes == t->ptr) &&
2993             (h0->ptr == h0->ptr0)) {
2994                 uint32_t offset;
2995
2996                 TRACE("Emit handler: header encapsulation.\n");
2997
2998                 offset = h0->n_bytes + h1->n_bytes;
2999                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
3000                 t->pkt.offset -= offset;
3001                 t->pkt.length += offset;
3002
3003                 return;
3004         }
3005
3006         /* Header insertion. */
3007         /* TBD */
3008
3009         /* Header extraction. */
3010         /* TBD */
3011
3012         /* For any other case. */
3013         TRACE("Emit handler: complex case.\n");
3014
3015         for (i = 0; i < t->n_headers_out; i++) {
3016                 struct header_out_runtime *h = &t->headers_out[i];
3017
3018                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
3019                 offset += h->n_bytes;
3020         }
3021
3022         if (offset) {
3023                 memcpy(t->ptr - offset, t->header_out_storage, offset);
3024                 t->pkt.offset -= offset;
3025                 t->pkt.length += offset;
3026         }
3027 }
3028
3029 static inline void
3030 instr_tx_exec(struct rte_swx_pipeline *p);
3031
3032 static inline void
3033 instr_tx_exec(struct rte_swx_pipeline *p)
3034 {
3035         struct thread *t = &p->threads[p->thread_id];
3036         struct instruction *ip = t->ip;
3037         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
3038         struct port_out_runtime *port = &p->out[port_id];
3039         struct rte_swx_pkt *pkt = &t->pkt;
3040
3041         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
3042               p->thread_id,
3043               (uint32_t)port_id);
3044
3045         /* Headers. */
3046         emit_handler(t);
3047
3048         /* Packet. */
3049         port->pkt_tx(port->obj, pkt);
3050
3051         /* Thread. */
3052         thread_ip_reset(p, t);
3053         instr_rx_exec(p);
3054 }
3055
3056 static inline void
3057 instr_tx_i_exec(struct rte_swx_pipeline *p)
3058 {
3059         struct thread *t = &p->threads[p->thread_id];
3060         struct instruction *ip = t->ip;
3061         uint64_t port_id = ip->io.io.val;
3062         struct port_out_runtime *port = &p->out[port_id];
3063         struct rte_swx_pkt *pkt = &t->pkt;
3064
3065         TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
3066               p->thread_id,
3067               (uint32_t)port_id);
3068
3069         /* Headers. */
3070         emit_handler(t);
3071
3072         /* Packet. */
3073         port->pkt_tx(port->obj, pkt);
3074
3075         /* Thread. */
3076         thread_ip_reset(p, t);
3077         instr_rx_exec(p);
3078 }
3079
3080 /*
3081  * extract.
3082  */
3083 static int
3084 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
3085                             struct action *action,
3086                             char **tokens,
3087                             int n_tokens,
3088                             struct instruction *instr,
3089                             struct instruction_data *data __rte_unused)
3090 {
3091         struct header *h;
3092
3093         CHECK(!action, EINVAL);
3094         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
3095
3096         h = header_parse(p, tokens[1]);
3097         CHECK(h, EINVAL);
3098
3099         if (n_tokens == 2) {
3100                 CHECK(!h->st->var_size, EINVAL);
3101
3102                 instr->type = INSTR_HDR_EXTRACT;
3103                 instr->io.hdr.header_id[0] = h->id;
3104                 instr->io.hdr.struct_id[0] = h->struct_id;
3105                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3106         } else {
3107                 struct field *mf;
3108
3109                 CHECK(h->st->var_size, EINVAL);
3110
3111                 mf = metadata_field_parse(p, tokens[2]);
3112                 CHECK(mf, EINVAL);
3113                 CHECK(!mf->var_size, EINVAL);
3114
3115                 instr->type = INSTR_HDR_EXTRACT_M;
3116                 instr->io.io.offset = mf->offset / 8;
3117                 instr->io.io.n_bits = mf->n_bits;
3118                 instr->io.hdr.header_id[0] = h->id;
3119                 instr->io.hdr.struct_id[0] = h->struct_id;
3120                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
3121         }
3122
3123         return 0;
3124 }
3125
3126 static inline void
3127 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
3128
3129 static inline void
3130 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
3131 {
3132         struct thread *t = &p->threads[p->thread_id];
3133         struct instruction *ip = t->ip;
3134         uint64_t valid_headers = t->valid_headers;
3135         uint8_t *ptr = t->ptr;
3136         uint32_t offset = t->pkt.offset;
3137         uint32_t length = t->pkt.length;
3138         uint32_t i;
3139
3140         for (i = 0; i < n_extract; i++) {
3141                 uint32_t header_id = ip->io.hdr.header_id[i];
3142                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3143                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3144
3145                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
3146                       p->thread_id,
3147                       header_id,
3148                       n_bytes);
3149
3150                 /* Headers. */
3151                 t->structs[struct_id] = ptr;
3152                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3153
3154                 /* Packet. */
3155                 offset += n_bytes;
3156                 length -= n_bytes;
3157                 ptr += n_bytes;
3158         }
3159
3160         /* Headers. */
3161         t->valid_headers = valid_headers;
3162
3163         /* Packet. */
3164         t->pkt.offset = offset;
3165         t->pkt.length = length;
3166         t->ptr = ptr;
3167 }
3168
3169 static inline void
3170 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
3171 {
3172         __instr_hdr_extract_exec(p, 1);
3173
3174         /* Thread. */
3175         thread_ip_inc(p);
3176 }
3177
3178 static inline void
3179 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
3180 {
3181         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3182               p->thread_id);
3183
3184         __instr_hdr_extract_exec(p, 2);
3185
3186         /* Thread. */
3187         thread_ip_inc(p);
3188 }
3189
3190 static inline void
3191 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
3192 {
3193         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3194               p->thread_id);
3195
3196         __instr_hdr_extract_exec(p, 3);
3197
3198         /* Thread. */
3199         thread_ip_inc(p);
3200 }
3201
3202 static inline void
3203 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
3204 {
3205         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3206               p->thread_id);
3207
3208         __instr_hdr_extract_exec(p, 4);
3209
3210         /* Thread. */
3211         thread_ip_inc(p);
3212 }
3213
3214 static inline void
3215 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
3216 {
3217         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3218               p->thread_id);
3219
3220         __instr_hdr_extract_exec(p, 5);
3221
3222         /* Thread. */
3223         thread_ip_inc(p);
3224 }
3225
3226 static inline void
3227 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
3228 {
3229         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3230               p->thread_id);
3231
3232         __instr_hdr_extract_exec(p, 6);
3233
3234         /* Thread. */
3235         thread_ip_inc(p);
3236 }
3237
3238 static inline void
3239 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
3240 {
3241         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3242               p->thread_id);
3243
3244         __instr_hdr_extract_exec(p, 7);
3245
3246         /* Thread. */
3247         thread_ip_inc(p);
3248 }
3249
3250 static inline void
3251 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
3252 {
3253         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3254               p->thread_id);
3255
3256         __instr_hdr_extract_exec(p, 8);
3257
3258         /* Thread. */
3259         thread_ip_inc(p);
3260 }
3261
3262 static inline void
3263 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
3264 {
3265         struct thread *t = &p->threads[p->thread_id];
3266         struct instruction *ip = t->ip;
3267
3268         uint64_t valid_headers = t->valid_headers;
3269         uint8_t *ptr = t->ptr;
3270         uint32_t offset = t->pkt.offset;
3271         uint32_t length = t->pkt.length;
3272
3273         uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
3274         uint32_t header_id = ip->io.hdr.header_id[0];
3275         uint32_t struct_id = ip->io.hdr.struct_id[0];
3276         uint32_t n_bytes = ip->io.hdr.n_bytes[0];
3277
3278         struct header_runtime *h = &t->headers[header_id];
3279
3280         TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
3281               p->thread_id,
3282               header_id,
3283               n_bytes,
3284               n_bytes_last);
3285
3286         n_bytes += n_bytes_last;
3287
3288         /* Headers. */
3289         t->structs[struct_id] = ptr;
3290         t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3291         h->n_bytes = n_bytes;
3292
3293         /* Packet. */
3294         t->pkt.offset = offset + n_bytes;
3295         t->pkt.length = length - n_bytes;
3296         t->ptr = ptr + n_bytes;
3297
3298         /* Thread. */
3299         thread_ip_inc(p);
3300 }
3301
3302 /*
3303  * emit.
3304  */
3305 static int
3306 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
3307                          struct action *action __rte_unused,
3308                          char **tokens,
3309                          int n_tokens,
3310                          struct instruction *instr,
3311                          struct instruction_data *data __rte_unused)
3312 {
3313         struct header *h;
3314
3315         CHECK(n_tokens == 2, EINVAL);
3316
3317         h = header_parse(p, tokens[1]);
3318         CHECK(h, EINVAL);
3319
3320         instr->type = INSTR_HDR_EMIT;
3321         instr->io.hdr.header_id[0] = h->id;
3322         instr->io.hdr.struct_id[0] = h->struct_id;
3323         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3324         return 0;
3325 }
3326
3327 static inline void
3328 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
3329
3330 static inline void
3331 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
3332 {
3333         struct thread *t = &p->threads[p->thread_id];
3334         struct instruction *ip = t->ip;
3335         uint64_t valid_headers = t->valid_headers;
3336         uint32_t n_headers_out = t->n_headers_out;
3337         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
3338         uint8_t *ho_ptr = NULL;
3339         uint32_t ho_nbytes = 0, first = 1, i;
3340
3341         for (i = 0; i < n_emit; i++) {
3342                 uint32_t header_id = ip->io.hdr.header_id[i];
3343                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3344
3345                 struct header_runtime *hi = &t->headers[header_id];
3346                 uint8_t *hi_ptr0 = hi->ptr0;
3347                 uint32_t n_bytes = hi->n_bytes;
3348
3349                 uint8_t *hi_ptr = t->structs[struct_id];
3350
3351                 if (!MASK64_BIT_GET(valid_headers, header_id))
3352                         continue;
3353
3354                 TRACE("[Thread %2u]: emit header %u\n",
3355                       p->thread_id,
3356                       header_id);
3357
3358                 /* Headers. */
3359                 if (first) {
3360                         first = 0;
3361
3362                         if (!t->n_headers_out) {
3363                                 ho = &t->headers_out[0];
3364
3365                                 ho->ptr0 = hi_ptr0;
3366                                 ho->ptr = hi_ptr;
3367
3368                                 ho_ptr = hi_ptr;
3369                                 ho_nbytes = n_bytes;
3370
3371                                 n_headers_out = 1;
3372
3373                                 continue;
3374                         } else {
3375                                 ho_ptr = ho->ptr;
3376                                 ho_nbytes = ho->n_bytes;
3377                         }
3378                 }
3379
3380                 if (ho_ptr + ho_nbytes == hi_ptr) {
3381                         ho_nbytes += n_bytes;
3382                 } else {
3383                         ho->n_bytes = ho_nbytes;
3384
3385                         ho++;
3386                         ho->ptr0 = hi_ptr0;
3387                         ho->ptr = hi_ptr;
3388
3389                         ho_ptr = hi_ptr;
3390                         ho_nbytes = n_bytes;
3391
3392                         n_headers_out++;
3393                 }
3394         }
3395
3396         ho->n_bytes = ho_nbytes;
3397         t->n_headers_out = n_headers_out;
3398 }
3399
3400 static inline void
3401 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
3402 {
3403         __instr_hdr_emit_exec(p, 1);
3404
3405         /* Thread. */
3406         thread_ip_inc(p);
3407 }
3408
3409 static inline void
3410 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
3411 {
3412         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3413               p->thread_id);
3414
3415         __instr_hdr_emit_exec(p, 1);
3416         instr_tx_exec(p);
3417 }
3418
3419 static inline void
3420 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
3421 {
3422         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3423               p->thread_id);
3424
3425         __instr_hdr_emit_exec(p, 2);
3426         instr_tx_exec(p);
3427 }
3428
3429 static inline void
3430 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
3431 {
3432         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3433               p->thread_id);
3434
3435         __instr_hdr_emit_exec(p, 3);
3436         instr_tx_exec(p);
3437 }
3438
3439 static inline void
3440 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
3441 {
3442         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3443               p->thread_id);
3444
3445         __instr_hdr_emit_exec(p, 4);
3446         instr_tx_exec(p);
3447 }
3448
3449 static inline void
3450 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
3451 {
3452         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3453               p->thread_id);
3454
3455         __instr_hdr_emit_exec(p, 5);
3456         instr_tx_exec(p);
3457 }
3458
3459 static inline void
3460 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
3461 {
3462         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3463               p->thread_id);
3464
3465         __instr_hdr_emit_exec(p, 6);
3466         instr_tx_exec(p);
3467 }
3468
3469 static inline void
3470 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
3471 {
3472         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3473               p->thread_id);
3474
3475         __instr_hdr_emit_exec(p, 7);
3476         instr_tx_exec(p);
3477 }
3478
3479 static inline void
3480 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
3481 {
3482         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
3483               p->thread_id);
3484
3485         __instr_hdr_emit_exec(p, 8);
3486         instr_tx_exec(p);
3487 }
3488
3489 /*
3490  * validate.
3491  */
3492 static int
3493 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
3494                              struct action *action __rte_unused,
3495                              char **tokens,
3496                              int n_tokens,
3497                              struct instruction *instr,
3498                              struct instruction_data *data __rte_unused)
3499 {
3500         struct header *h;
3501
3502         CHECK(n_tokens == 2, EINVAL);
3503
3504         h = header_parse(p, tokens[1]);
3505         CHECK(h, EINVAL);
3506
3507         instr->type = INSTR_HDR_VALIDATE;
3508         instr->valid.header_id = h->id;
3509         return 0;
3510 }
3511
3512 static inline void
3513 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
3514 {
3515         struct thread *t = &p->threads[p->thread_id];
3516         struct instruction *ip = t->ip;
3517         uint32_t header_id = ip->valid.header_id;
3518
3519         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
3520
3521         /* Headers. */
3522         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
3523
3524         /* Thread. */
3525         thread_ip_inc(p);
3526 }
3527
3528 /*
3529  * invalidate.
3530  */
3531 static int
3532 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
3533                                struct action *action __rte_unused,
3534                                char **tokens,
3535                                int n_tokens,
3536                                struct instruction *instr,
3537                                struct instruction_data *data __rte_unused)
3538 {
3539         struct header *h;
3540
3541         CHECK(n_tokens == 2, EINVAL);
3542
3543         h = header_parse(p, tokens[1]);
3544         CHECK(h, EINVAL);
3545
3546         instr->type = INSTR_HDR_INVALIDATE;
3547         instr->valid.header_id = h->id;
3548         return 0;
3549 }
3550
3551 static inline void
3552 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3553 {
3554         struct thread *t = &p->threads[p->thread_id];
3555         struct instruction *ip = t->ip;
3556         uint32_t header_id = ip->valid.header_id;
3557
3558         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3559
3560         /* Headers. */
3561         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3562
3563         /* Thread. */
3564         thread_ip_inc(p);
3565 }
3566
3567 /*
3568  * table.
3569  */
3570 static struct table *
3571 table_find(struct rte_swx_pipeline *p, const char *name);
3572
3573 static struct selector *
3574 selector_find(struct rte_swx_pipeline *p, const char *name);
3575
3576 static int
3577 instr_table_translate(struct rte_swx_pipeline *p,
3578                       struct action *action,
3579                       char **tokens,
3580                       int n_tokens,
3581                       struct instruction *instr,
3582                       struct instruction_data *data __rte_unused)
3583 {
3584         struct table *t;
3585         struct selector *s;
3586
3587         CHECK(!action, EINVAL);
3588         CHECK(n_tokens == 2, EINVAL);
3589
3590         t = table_find(p, tokens[1]);
3591         if (t) {
3592                 instr->type = INSTR_TABLE;
3593                 instr->table.table_id = t->id;
3594                 return 0;
3595         }
3596
3597         s = selector_find(p, tokens[1]);
3598         if (s) {
3599                 instr->type = INSTR_SELECTOR;
3600                 instr->table.table_id = s->id;
3601                 return 0;
3602         }
3603
3604         CHECK(0, EINVAL);
3605 }
3606
3607 static inline void
3608 instr_table_exec(struct rte_swx_pipeline *p)
3609 {
3610         struct thread *t = &p->threads[p->thread_id];
3611         struct instruction *ip = t->ip;
3612         uint32_t table_id = ip->table.table_id;
3613         struct rte_swx_table_state *ts = &t->table_state[table_id];
3614         struct table_runtime *table = &t->tables[table_id];
3615         struct table_statistics *stats = &p->table_stats[table_id];
3616         uint64_t action_id, n_pkts_hit, n_pkts_action;
3617         uint8_t *action_data;
3618         int done, hit;
3619
3620         /* Table. */
3621         done = table->func(ts->obj,
3622                            table->mailbox,
3623                            table->key,
3624                            &action_id,
3625                            &action_data,
3626                            &hit);
3627         if (!done) {
3628                 /* Thread. */
3629                 TRACE("[Thread %2u] table %u (not finalized)\n",
3630                       p->thread_id,
3631                       table_id);
3632
3633                 thread_yield(p);
3634                 return;
3635         }
3636
3637         action_id = hit ? action_id : ts->default_action_id;
3638         action_data = hit ? action_data : ts->default_action_data;
3639         n_pkts_hit = stats->n_pkts_hit[hit];
3640         n_pkts_action = stats->n_pkts_action[action_id];
3641
3642         TRACE("[Thread %2u] table %u (%s, action %u)\n",
3643               p->thread_id,
3644               table_id,
3645               hit ? "hit" : "miss",
3646               (uint32_t)action_id);
3647
3648         t->action_id = action_id;
3649         t->structs[0] = action_data;
3650         t->hit = hit;
3651         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
3652         stats->n_pkts_action[action_id] = n_pkts_action + 1;
3653
3654         /* Thread. */
3655         thread_ip_action_call(p, t, action_id);
3656 }
3657
3658 static inline void
3659 instr_selector_exec(struct rte_swx_pipeline *p)
3660 {
3661         struct thread *t = &p->threads[p->thread_id];
3662         struct instruction *ip = t->ip;
3663         uint32_t selector_id = ip->table.table_id;
3664         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
3665         struct selector_runtime *selector = &t->selectors[selector_id];
3666         struct selector_statistics *stats = &p->selector_stats[selector_id];
3667         uint64_t n_pkts = stats->n_pkts;
3668         int done;
3669
3670         /* Table. */
3671         done = rte_swx_table_selector_select(ts->obj,
3672                            selector->mailbox,
3673                            selector->group_id_buffer,
3674                            selector->selector_buffer,
3675                            selector->member_id_buffer);
3676         if (!done) {
3677                 /* Thread. */
3678                 TRACE("[Thread %2u] selector %u (not finalized)\n",
3679                       p->thread_id,
3680                       selector_id);
3681
3682                 thread_yield(p);
3683                 return;
3684         }
3685
3686
3687         TRACE("[Thread %2u] selector %u\n",
3688               p->thread_id,
3689               selector_id);
3690
3691         stats->n_pkts = n_pkts + 1;
3692
3693         /* Thread. */
3694         thread_ip_inc(p);
3695 }
3696
3697 /*
3698  * extern.
3699  */
3700 static int
3701 instr_extern_translate(struct rte_swx_pipeline *p,
3702                        struct action *action __rte_unused,
3703                        char **tokens,
3704                        int n_tokens,
3705                        struct instruction *instr,
3706                        struct instruction_data *data __rte_unused)
3707 {
3708         char *token = tokens[1];
3709
3710         CHECK(n_tokens == 2, EINVAL);
3711
3712         if (token[0] == 'e') {
3713                 struct extern_obj *obj;
3714                 struct extern_type_member_func *func;
3715
3716                 func = extern_obj_member_func_parse(p, token, &obj);
3717                 CHECK(func, EINVAL);
3718
3719                 instr->type = INSTR_EXTERN_OBJ;
3720                 instr->ext_obj.ext_obj_id = obj->id;
3721                 instr->ext_obj.func_id = func->id;
3722
3723                 return 0;
3724         }
3725
3726         if (token[0] == 'f') {
3727                 struct extern_func *func;
3728
3729                 func = extern_func_parse(p, token);
3730                 CHECK(func, EINVAL);
3731
3732                 instr->type = INSTR_EXTERN_FUNC;
3733                 instr->ext_func.ext_func_id = func->id;
3734
3735                 return 0;
3736         }
3737
3738         CHECK(0, EINVAL);
3739 }
3740
3741 static inline void
3742 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3743 {
3744         struct thread *t = &p->threads[p->thread_id];
3745         struct instruction *ip = t->ip;
3746         uint32_t obj_id = ip->ext_obj.ext_obj_id;
3747         uint32_t func_id = ip->ext_obj.func_id;
3748         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3749         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3750
3751         TRACE("[Thread %2u] extern obj %u member func %u\n",
3752               p->thread_id,
3753               obj_id,
3754               func_id);
3755
3756         /* Extern object member function execute. */
3757         uint32_t done = func(obj->obj, obj->mailbox);
3758
3759         /* Thread. */
3760         thread_ip_inc_cond(t, done);
3761         thread_yield_cond(p, done ^ 1);
3762 }
3763
3764 static inline void
3765 instr_extern_func_exec(struct rte_swx_pipeline *p)
3766 {
3767         struct thread *t = &p->threads[p->thread_id];
3768         struct instruction *ip = t->ip;
3769         uint32_t ext_func_id = ip->ext_func.ext_func_id;
3770         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3771         rte_swx_extern_func_t func = ext_func->func;
3772
3773         TRACE("[Thread %2u] extern func %u\n",
3774               p->thread_id,
3775               ext_func_id);
3776
3777         /* Extern function execute. */
3778         uint32_t done = func(ext_func->mailbox);
3779
3780         /* Thread. */
3781         thread_ip_inc_cond(t, done);
3782         thread_yield_cond(p, done ^ 1);
3783 }
3784
3785 /*
3786  * mov.
3787  */
3788 static int
3789 instr_mov_translate(struct rte_swx_pipeline *p,
3790                     struct action *action,
3791                     char **tokens,
3792                     int n_tokens,
3793                     struct instruction *instr,
3794                     struct instruction_data *data __rte_unused)
3795 {
3796         char *dst = tokens[1], *src = tokens[2];
3797         struct field *fdst, *fsrc;
3798         uint64_t src_val;
3799         uint32_t dst_struct_id = 0, src_struct_id = 0;
3800
3801         CHECK(n_tokens == 3, EINVAL);
3802
3803         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3804         CHECK(fdst, EINVAL);
3805         CHECK(!fdst->var_size, EINVAL);
3806
3807         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
3808         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3809         if (fsrc) {
3810                 CHECK(!fsrc->var_size, EINVAL);
3811
3812                 instr->type = INSTR_MOV;
3813                 if (dst[0] != 'h' && src[0] == 'h')
3814                         instr->type = INSTR_MOV_MH;
3815                 if (dst[0] == 'h' && src[0] != 'h')
3816                         instr->type = INSTR_MOV_HM;
3817                 if (dst[0] == 'h' && src[0] == 'h')
3818                         instr->type = INSTR_MOV_HH;
3819
3820                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3821                 instr->mov.dst.n_bits = fdst->n_bits;
3822                 instr->mov.dst.offset = fdst->offset / 8;
3823                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3824                 instr->mov.src.n_bits = fsrc->n_bits;
3825                 instr->mov.src.offset = fsrc->offset / 8;
3826                 return 0;
3827         }
3828
3829         /* MOV_I. */
3830         src_val = strtoull(src, &src, 0);
3831         CHECK(!src[0], EINVAL);
3832
3833         if (dst[0] == 'h')
3834                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3835
3836         instr->type = INSTR_MOV_I;
3837         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3838         instr->mov.dst.n_bits = fdst->n_bits;
3839         instr->mov.dst.offset = fdst->offset / 8;
3840         instr->mov.src_val = src_val;
3841         return 0;
3842 }
3843
3844 static inline void
3845 instr_mov_exec(struct rte_swx_pipeline *p)
3846 {
3847         struct thread *t = &p->threads[p->thread_id];
3848         struct instruction *ip = t->ip;
3849
3850         TRACE("[Thread %2u] mov\n",
3851               p->thread_id);
3852
3853         MOV(t, ip);
3854
3855         /* Thread. */
3856         thread_ip_inc(p);
3857 }
3858
3859 static inline void
3860 instr_mov_mh_exec(struct rte_swx_pipeline *p)
3861 {
3862         struct thread *t = &p->threads[p->thread_id];
3863         struct instruction *ip = t->ip;
3864
3865         TRACE("[Thread %2u] mov (mh)\n",
3866               p->thread_id);
3867
3868         MOV_MH(t, ip);
3869
3870         /* Thread. */
3871         thread_ip_inc(p);
3872 }
3873
3874 static inline void
3875 instr_mov_hm_exec(struct rte_swx_pipeline *p)
3876 {
3877         struct thread *t = &p->threads[p->thread_id];
3878         struct instruction *ip = t->ip;
3879
3880         TRACE("[Thread %2u] mov (hm)\n",
3881               p->thread_id);
3882
3883         MOV_HM(t, ip);
3884
3885         /* Thread. */
3886         thread_ip_inc(p);
3887 }
3888
3889 static inline void
3890 instr_mov_hh_exec(struct rte_swx_pipeline *p)
3891 {
3892         struct thread *t = &p->threads[p->thread_id];
3893         struct instruction *ip = t->ip;
3894
3895         TRACE("[Thread %2u] mov (hh)\n",
3896               p->thread_id);
3897
3898         MOV_HH(t, ip);
3899
3900         /* Thread. */
3901         thread_ip_inc(p);
3902 }
3903
3904 static inline void
3905 instr_mov_i_exec(struct rte_swx_pipeline *p)
3906 {
3907         struct thread *t = &p->threads[p->thread_id];
3908         struct instruction *ip = t->ip;
3909
3910         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
3911               p->thread_id,
3912               ip->mov.src_val);
3913
3914         MOV_I(t, ip);
3915
3916         /* Thread. */
3917         thread_ip_inc(p);
3918 }
3919
3920 /*
3921  * dma.
3922  */
3923 static inline void
3924 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3925
3926 static inline void
3927 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3928 {
3929         struct thread *t = &p->threads[p->thread_id];
3930         struct instruction *ip = t->ip;
3931         uint8_t *action_data = t->structs[0];
3932         uint64_t valid_headers = t->valid_headers;
3933         uint32_t i;
3934
3935         for (i = 0; i < n_dma; i++) {
3936                 uint32_t header_id = ip->dma.dst.header_id[i];
3937                 uint32_t struct_id = ip->dma.dst.struct_id[i];
3938                 uint32_t offset = ip->dma.src.offset[i];
3939                 uint32_t n_bytes = ip->dma.n_bytes[i];
3940
3941                 struct header_runtime *h = &t->headers[header_id];
3942                 uint8_t *h_ptr0 = h->ptr0;
3943                 uint8_t *h_ptr = t->structs[struct_id];
3944
3945                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3946                         h_ptr : h_ptr0;
3947                 void *src = &action_data[offset];
3948
3949                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3950
3951                 /* Headers. */
3952                 memcpy(dst, src, n_bytes);
3953                 t->structs[struct_id] = dst;
3954                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3955         }
3956
3957         t->valid_headers = valid_headers;
3958 }
3959
3960 static inline void
3961 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3962 {
3963         __instr_dma_ht_exec(p, 1);
3964
3965         /* Thread. */
3966         thread_ip_inc(p);
3967 }
3968
3969 static inline void
3970 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3971 {
3972         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3973               p->thread_id);
3974
3975         __instr_dma_ht_exec(p, 2);
3976
3977         /* Thread. */
3978         thread_ip_inc(p);
3979 }
3980
3981 static inline void
3982 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3983 {
3984         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3985               p->thread_id);
3986
3987         __instr_dma_ht_exec(p, 3);
3988
3989         /* Thread. */
3990         thread_ip_inc(p);
3991 }
3992
3993 static inline void
3994 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3995 {
3996         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3997               p->thread_id);
3998
3999         __instr_dma_ht_exec(p, 4);
4000
4001         /* Thread. */
4002         thread_ip_inc(p);
4003 }
4004
4005 static inline void
4006 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
4007 {
4008         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
4009               p->thread_id);
4010
4011         __instr_dma_ht_exec(p, 5);
4012
4013         /* Thread. */
4014         thread_ip_inc(p);
4015 }
4016
4017 static inline void
4018 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
4019 {
4020         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
4021               p->thread_id);
4022
4023         __instr_dma_ht_exec(p, 6);
4024
4025         /* Thread. */
4026         thread_ip_inc(p);
4027 }
4028
4029 static inline void
4030 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
4031 {
4032         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
4033               p->thread_id);
4034
4035         __instr_dma_ht_exec(p, 7);
4036
4037         /* Thread. */
4038         thread_ip_inc(p);
4039 }
4040
4041 static inline void
4042 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
4043 {
4044         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
4045               p->thread_id);
4046
4047         __instr_dma_ht_exec(p, 8);
4048
4049         /* Thread. */
4050         thread_ip_inc(p);
4051 }
4052
4053 /*
4054  * alu.
4055  */
4056 static int
4057 instr_alu_add_translate(struct rte_swx_pipeline *p,
4058                         struct action *action,
4059                         char **tokens,
4060                         int n_tokens,
4061                         struct instruction *instr,
4062                         struct instruction_data *data __rte_unused)
4063 {
4064         char *dst = tokens[1], *src = tokens[2];
4065         struct field *fdst, *fsrc;
4066         uint64_t src_val;
4067         uint32_t dst_struct_id = 0, src_struct_id = 0;
4068
4069         CHECK(n_tokens == 3, EINVAL);
4070
4071         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4072         CHECK(fdst, EINVAL);
4073         CHECK(!fdst->var_size, EINVAL);
4074
4075         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
4076         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4077         if (fsrc) {
4078                 CHECK(!fsrc->var_size, EINVAL);
4079
4080                 instr->type = INSTR_ALU_ADD;
4081                 if (dst[0] == 'h' && src[0] != 'h')
4082                         instr->type = INSTR_ALU_ADD_HM;
4083                 if (dst[0] != 'h' && src[0] == 'h')
4084                         instr->type = INSTR_ALU_ADD_MH;
4085                 if (dst[0] == 'h' && src[0] == 'h')
4086                         instr->type = INSTR_ALU_ADD_HH;
4087
4088                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4089                 instr->alu.dst.n_bits = fdst->n_bits;
4090                 instr->alu.dst.offset = fdst->offset / 8;
4091                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4092                 instr->alu.src.n_bits = fsrc->n_bits;
4093                 instr->alu.src.offset = fsrc->offset / 8;
4094                 return 0;
4095         }
4096
4097         /* ADD_MI, ADD_HI. */
4098         src_val = strtoull(src, &src, 0);
4099         CHECK(!src[0], EINVAL);
4100
4101         instr->type = INSTR_ALU_ADD_MI;
4102         if (dst[0] == 'h')
4103                 instr->type = INSTR_ALU_ADD_HI;
4104
4105         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4106         instr->alu.dst.n_bits = fdst->n_bits;
4107         instr->alu.dst.offset = fdst->offset / 8;
4108         instr->alu.src_val = src_val;
4109         return 0;
4110 }
4111
4112 static int
4113 instr_alu_sub_translate(struct rte_swx_pipeline *p,
4114                         struct action *action,
4115                         char **tokens,
4116                         int n_tokens,
4117                         struct instruction *instr,
4118                         struct instruction_data *data __rte_unused)
4119 {
4120         char *dst = tokens[1], *src = tokens[2];
4121         struct field *fdst, *fsrc;
4122         uint64_t src_val;
4123         uint32_t dst_struct_id = 0, src_struct_id = 0;
4124
4125         CHECK(n_tokens == 3, EINVAL);
4126
4127         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4128         CHECK(fdst, EINVAL);
4129         CHECK(!fdst->var_size, EINVAL);
4130
4131         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
4132         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4133         if (fsrc) {
4134                 CHECK(!fsrc->var_size, EINVAL);
4135
4136                 instr->type = INSTR_ALU_SUB;
4137                 if (dst[0] == 'h' && src[0] != 'h')
4138                         instr->type = INSTR_ALU_SUB_HM;
4139                 if (dst[0] != 'h' && src[0] == 'h')
4140                         instr->type = INSTR_ALU_SUB_MH;
4141                 if (dst[0] == 'h' && src[0] == 'h')
4142                         instr->type = INSTR_ALU_SUB_HH;
4143
4144                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4145                 instr->alu.dst.n_bits = fdst->n_bits;
4146                 instr->alu.dst.offset = fdst->offset / 8;
4147                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4148                 instr->alu.src.n_bits = fsrc->n_bits;
4149                 instr->alu.src.offset = fsrc->offset / 8;
4150                 return 0;
4151         }
4152
4153         /* SUB_MI, SUB_HI. */
4154         src_val = strtoull(src, &src, 0);
4155         CHECK(!src[0], EINVAL);
4156
4157         instr->type = INSTR_ALU_SUB_MI;
4158         if (dst[0] == 'h')
4159                 instr->type = INSTR_ALU_SUB_HI;
4160
4161         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4162         instr->alu.dst.n_bits = fdst->n_bits;
4163         instr->alu.dst.offset = fdst->offset / 8;
4164         instr->alu.src_val = src_val;
4165         return 0;
4166 }
4167
4168 static int
4169 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
4170                           struct action *action __rte_unused,
4171                           char **tokens,
4172                           int n_tokens,
4173                           struct instruction *instr,
4174                           struct instruction_data *data __rte_unused)
4175 {
4176         char *dst = tokens[1], *src = tokens[2];
4177         struct header *hdst, *hsrc;
4178         struct field *fdst, *fsrc;
4179
4180         CHECK(n_tokens == 3, EINVAL);
4181
4182         fdst = header_field_parse(p, dst, &hdst);
4183         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4184         CHECK(!fdst->var_size, EINVAL);
4185
4186         /* CKADD_FIELD. */
4187         fsrc = header_field_parse(p, src, &hsrc);
4188         if (fsrc) {
4189                 CHECK(!fsrc->var_size, EINVAL);
4190
4191                 instr->type = INSTR_ALU_CKADD_FIELD;
4192                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4193                 instr->alu.dst.n_bits = fdst->n_bits;
4194                 instr->alu.dst.offset = fdst->offset / 8;
4195                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4196                 instr->alu.src.n_bits = fsrc->n_bits;
4197                 instr->alu.src.offset = fsrc->offset / 8;
4198                 return 0;
4199         }
4200
4201         /* CKADD_STRUCT, CKADD_STRUCT20. */
4202         hsrc = header_parse(p, src);
4203         CHECK(hsrc, EINVAL);
4204         CHECK(!hsrc->st->var_size, EINVAL);
4205
4206         instr->type = INSTR_ALU_CKADD_STRUCT;
4207         if ((hsrc->st->n_bits / 8) == 20)
4208                 instr->type = INSTR_ALU_CKADD_STRUCT20;
4209
4210         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4211         instr->alu.dst.n_bits = fdst->n_bits;
4212         instr->alu.dst.offset = fdst->offset / 8;
4213         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4214         instr->alu.src.n_bits = hsrc->st->n_bits;
4215         instr->alu.src.offset = 0; /* Unused. */
4216         return 0;
4217 }
4218
4219 static int
4220 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
4221                           struct action *action __rte_unused,
4222                           char **tokens,
4223                           int n_tokens,
4224                           struct instruction *instr,
4225                           struct instruction_data *data __rte_unused)
4226 {
4227         char *dst = tokens[1], *src = tokens[2];
4228         struct header *hdst, *hsrc;
4229         struct field *fdst, *fsrc;
4230
4231         CHECK(n_tokens == 3, EINVAL);
4232
4233         fdst = header_field_parse(p, dst, &hdst);
4234         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4235         CHECK(!fdst->var_size, EINVAL);
4236
4237         fsrc = header_field_parse(p, src, &hsrc);
4238         CHECK(fsrc, EINVAL);
4239         CHECK(!fsrc->var_size, EINVAL);
4240
4241         instr->type = INSTR_ALU_CKSUB_FIELD;
4242         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4243         instr->alu.dst.n_bits = fdst->n_bits;
4244         instr->alu.dst.offset = fdst->offset / 8;
4245         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4246         instr->alu.src.n_bits = fsrc->n_bits;
4247         instr->alu.src.offset = fsrc->offset / 8;
4248         return 0;
4249 }
4250
4251 static int
4252 instr_alu_shl_translate(struct rte_swx_pipeline *p,
4253                         struct action *action,
4254                         char **tokens,
4255                         int n_tokens,
4256                         struct instruction *instr,
4257                         struct instruction_data *data __rte_unused)
4258 {
4259         char *dst = tokens[1], *src = tokens[2];
4260         struct field *fdst, *fsrc;
4261         uint64_t src_val;
4262         uint32_t dst_struct_id = 0, src_struct_id = 0;
4263
4264         CHECK(n_tokens == 3, EINVAL);
4265
4266         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4267         CHECK(fdst, EINVAL);
4268         CHECK(!fdst->var_size, EINVAL);
4269
4270         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
4271         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4272         if (fsrc) {
4273                 CHECK(!fsrc->var_size, EINVAL);
4274
4275                 instr->type = INSTR_ALU_SHL;
4276                 if (dst[0] == 'h' && src[0] != 'h')
4277                         instr->type = INSTR_ALU_SHL_HM;
4278                 if (dst[0] != 'h' && src[0] == 'h')
4279                         instr->type = INSTR_ALU_SHL_MH;
4280                 if (dst[0] == 'h' && src[0] == 'h')
4281                         instr->type = INSTR_ALU_SHL_HH;
4282
4283                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4284                 instr->alu.dst.n_bits = fdst->n_bits;
4285                 instr->alu.dst.offset = fdst->offset / 8;
4286                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4287                 instr->alu.src.n_bits = fsrc->n_bits;
4288                 instr->alu.src.offset = fsrc->offset / 8;
4289                 return 0;
4290         }
4291
4292         /* SHL_MI, SHL_HI. */
4293         src_val = strtoull(src, &src, 0);
4294         CHECK(!src[0], EINVAL);
4295
4296         instr->type = INSTR_ALU_SHL_MI;
4297         if (dst[0] == 'h')
4298                 instr->type = INSTR_ALU_SHL_HI;
4299
4300         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4301         instr->alu.dst.n_bits = fdst->n_bits;
4302         instr->alu.dst.offset = fdst->offset / 8;
4303         instr->alu.src_val = src_val;
4304         return 0;
4305 }
4306
4307 static int
4308 instr_alu_shr_translate(struct rte_swx_pipeline *p,
4309                         struct action *action,
4310                         char **tokens,
4311                         int n_tokens,
4312                         struct instruction *instr,
4313                         struct instruction_data *data __rte_unused)
4314 {
4315         char *dst = tokens[1], *src = tokens[2];
4316         struct field *fdst, *fsrc;
4317         uint64_t src_val;
4318         uint32_t dst_struct_id = 0, src_struct_id = 0;
4319
4320         CHECK(n_tokens == 3, EINVAL);
4321
4322         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4323         CHECK(fdst, EINVAL);
4324         CHECK(!fdst->var_size, EINVAL);
4325
4326         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
4327         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4328         if (fsrc) {
4329                 CHECK(!fsrc->var_size, EINVAL);
4330
4331                 instr->type = INSTR_ALU_SHR;
4332                 if (dst[0] == 'h' && src[0] != 'h')
4333                         instr->type = INSTR_ALU_SHR_HM;
4334                 if (dst[0] != 'h' && src[0] == 'h')
4335                         instr->type = INSTR_ALU_SHR_MH;
4336                 if (dst[0] == 'h' && src[0] == 'h')
4337                         instr->type = INSTR_ALU_SHR_HH;
4338
4339                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4340                 instr->alu.dst.n_bits = fdst->n_bits;
4341                 instr->alu.dst.offset = fdst->offset / 8;
4342                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4343                 instr->alu.src.n_bits = fsrc->n_bits;
4344                 instr->alu.src.offset = fsrc->offset / 8;
4345                 return 0;
4346         }
4347
4348         /* SHR_MI, SHR_HI. */
4349         src_val = strtoull(src, &src, 0);
4350         CHECK(!src[0], EINVAL);
4351
4352         instr->type = INSTR_ALU_SHR_MI;
4353         if (dst[0] == 'h')
4354                 instr->type = INSTR_ALU_SHR_HI;
4355
4356         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4357         instr->alu.dst.n_bits = fdst->n_bits;
4358         instr->alu.dst.offset = fdst->offset / 8;
4359         instr->alu.src_val = src_val;
4360         return 0;
4361 }
4362
4363 static int
4364 instr_alu_and_translate(struct rte_swx_pipeline *p,
4365                         struct action *action,
4366                         char **tokens,
4367                         int n_tokens,
4368                         struct instruction *instr,
4369                         struct instruction_data *data __rte_unused)
4370 {
4371         char *dst = tokens[1], *src = tokens[2];
4372         struct field *fdst, *fsrc;
4373         uint64_t src_val;
4374         uint32_t dst_struct_id = 0, src_struct_id = 0;
4375
4376         CHECK(n_tokens == 3, EINVAL);
4377
4378         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4379         CHECK(fdst, EINVAL);
4380         CHECK(!fdst->var_size, EINVAL);
4381
4382         /* AND, AND_MH, AND_HM, AND_HH. */
4383         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4384         if (fsrc) {
4385                 CHECK(!fsrc->var_size, EINVAL);
4386
4387                 instr->type = INSTR_ALU_AND;
4388                 if (dst[0] != 'h' && src[0] == 'h')
4389                         instr->type = INSTR_ALU_AND_MH;
4390                 if (dst[0] == 'h' && src[0] != 'h')
4391                         instr->type = INSTR_ALU_AND_HM;
4392                 if (dst[0] == 'h' && src[0] == 'h')
4393                         instr->type = INSTR_ALU_AND_HH;
4394
4395                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4396                 instr->alu.dst.n_bits = fdst->n_bits;
4397                 instr->alu.dst.offset = fdst->offset / 8;
4398                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4399                 instr->alu.src.n_bits = fsrc->n_bits;
4400                 instr->alu.src.offset = fsrc->offset / 8;
4401                 return 0;
4402         }
4403
4404         /* AND_I. */
4405         src_val = strtoull(src, &src, 0);
4406         CHECK(!src[0], EINVAL);
4407
4408         if (dst[0] == 'h')
4409                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4410
4411         instr->type = INSTR_ALU_AND_I;
4412         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4413         instr->alu.dst.n_bits = fdst->n_bits;
4414         instr->alu.dst.offset = fdst->offset / 8;
4415         instr->alu.src_val = src_val;
4416         return 0;
4417 }
4418
4419 static int
4420 instr_alu_or_translate(struct rte_swx_pipeline *p,
4421                        struct action *action,
4422                        char **tokens,
4423                        int n_tokens,
4424                        struct instruction *instr,
4425                        struct instruction_data *data __rte_unused)
4426 {
4427         char *dst = tokens[1], *src = tokens[2];
4428         struct field *fdst, *fsrc;
4429         uint64_t src_val;
4430         uint32_t dst_struct_id = 0, src_struct_id = 0;
4431
4432         CHECK(n_tokens == 3, EINVAL);
4433
4434         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4435         CHECK(fdst, EINVAL);
4436         CHECK(!fdst->var_size, EINVAL);
4437
4438         /* OR, OR_MH, OR_HM, OR_HH. */
4439         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4440         if (fsrc) {
4441                 CHECK(!fsrc->var_size, EINVAL);
4442
4443                 instr->type = INSTR_ALU_OR;
4444                 if (dst[0] != 'h' && src[0] == 'h')
4445                         instr->type = INSTR_ALU_OR_MH;
4446                 if (dst[0] == 'h' && src[0] != 'h')
4447                         instr->type = INSTR_ALU_OR_HM;
4448                 if (dst[0] == 'h' && src[0] == 'h')
4449                         instr->type = INSTR_ALU_OR_HH;
4450
4451                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4452                 instr->alu.dst.n_bits = fdst->n_bits;
4453                 instr->alu.dst.offset = fdst->offset / 8;
4454                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4455                 instr->alu.src.n_bits = fsrc->n_bits;
4456                 instr->alu.src.offset = fsrc->offset / 8;
4457                 return 0;
4458         }
4459
4460         /* OR_I. */
4461         src_val = strtoull(src, &src, 0);
4462         CHECK(!src[0], EINVAL);
4463
4464         if (dst[0] == 'h')
4465                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4466
4467         instr->type = INSTR_ALU_OR_I;
4468         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4469         instr->alu.dst.n_bits = fdst->n_bits;
4470         instr->alu.dst.offset = fdst->offset / 8;
4471         instr->alu.src_val = src_val;
4472         return 0;
4473 }
4474
4475 static int
4476 instr_alu_xor_translate(struct rte_swx_pipeline *p,
4477                         struct action *action,
4478                         char **tokens,
4479                         int n_tokens,
4480                         struct instruction *instr,
4481                         struct instruction_data *data __rte_unused)
4482 {
4483         char *dst = tokens[1], *src = tokens[2];
4484         struct field *fdst, *fsrc;
4485         uint64_t src_val;
4486         uint32_t dst_struct_id = 0, src_struct_id = 0;
4487
4488         CHECK(n_tokens == 3, EINVAL);
4489
4490         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4491         CHECK(fdst, EINVAL);
4492         CHECK(!fdst->var_size, EINVAL);
4493
4494         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
4495         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4496         if (fsrc) {
4497                 CHECK(!fsrc->var_size, EINVAL);
4498
4499                 instr->type = INSTR_ALU_XOR;
4500                 if (dst[0] != 'h' && src[0] == 'h')
4501                         instr->type = INSTR_ALU_XOR_MH;
4502                 if (dst[0] == 'h' && src[0] != 'h')
4503                         instr->type = INSTR_ALU_XOR_HM;
4504                 if (dst[0] == 'h' && src[0] == 'h')
4505                         instr->type = INSTR_ALU_XOR_HH;
4506
4507                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4508                 instr->alu.dst.n_bits = fdst->n_bits;
4509                 instr->alu.dst.offset = fdst->offset / 8;
4510                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4511                 instr->alu.src.n_bits = fsrc->n_bits;
4512                 instr->alu.src.offset = fsrc->offset / 8;
4513                 return 0;
4514         }
4515
4516         /* XOR_I. */
4517         src_val = strtoull(src, &src, 0);
4518         CHECK(!src[0], EINVAL);
4519
4520         if (dst[0] == 'h')
4521                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4522
4523         instr->type = INSTR_ALU_XOR_I;
4524         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4525         instr->alu.dst.n_bits = fdst->n_bits;
4526         instr->alu.dst.offset = fdst->offset / 8;
4527         instr->alu.src_val = src_val;
4528         return 0;
4529 }
4530
4531 static inline void
4532 instr_alu_add_exec(struct rte_swx_pipeline *p)
4533 {
4534         struct thread *t = &p->threads[p->thread_id];
4535         struct instruction *ip = t->ip;
4536
4537         TRACE("[Thread %2u] add\n", p->thread_id);
4538
4539         /* Structs. */
4540         ALU(t, ip, +);
4541
4542         /* Thread. */
4543         thread_ip_inc(p);
4544 }
4545
4546 static inline void
4547 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
4548 {
4549         struct thread *t = &p->threads[p->thread_id];
4550         struct instruction *ip = t->ip;
4551
4552         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
4553
4554         /* Structs. */
4555         ALU_MH(t, ip, +);
4556
4557         /* Thread. */
4558         thread_ip_inc(p);
4559 }
4560
4561 static inline void
4562 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
4563 {
4564         struct thread *t = &p->threads[p->thread_id];
4565         struct instruction *ip = t->ip;
4566
4567         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
4568
4569         /* Structs. */
4570         ALU_HM(t, ip, +);
4571
4572         /* Thread. */
4573         thread_ip_inc(p);
4574 }
4575
4576 static inline void
4577 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
4578 {
4579         struct thread *t = &p->threads[p->thread_id];
4580         struct instruction *ip = t->ip;
4581
4582         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
4583
4584         /* Structs. */
4585         ALU_HH(t, ip, +);
4586
4587         /* Thread. */
4588         thread_ip_inc(p);
4589 }
4590
4591 static inline void
4592 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
4593 {
4594         struct thread *t = &p->threads[p->thread_id];
4595         struct instruction *ip = t->ip;
4596
4597         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
4598
4599         /* Structs. */
4600         ALU_MI(t, ip, +);
4601
4602         /* Thread. */
4603         thread_ip_inc(p);
4604 }
4605
4606 static inline void
4607 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
4608 {
4609         struct thread *t = &p->threads[p->thread_id];
4610         struct instruction *ip = t->ip;
4611
4612         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
4613
4614         /* Structs. */
4615         ALU_HI(t, ip, +);
4616
4617         /* Thread. */
4618         thread_ip_inc(p);
4619 }
4620
4621 static inline void
4622 instr_alu_sub_exec(struct rte_swx_pipeline *p)
4623 {
4624         struct thread *t = &p->threads[p->thread_id];
4625         struct instruction *ip = t->ip;
4626
4627         TRACE("[Thread %2u] sub\n", p->thread_id);
4628
4629         /* Structs. */
4630         ALU(t, ip, -);
4631
4632         /* Thread. */
4633         thread_ip_inc(p);
4634 }
4635
4636 static inline void
4637 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
4638 {
4639         struct thread *t = &p->threads[p->thread_id];
4640         struct instruction *ip = t->ip;
4641
4642         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
4643
4644         /* Structs. */
4645         ALU_MH(t, ip, -);
4646
4647         /* Thread. */
4648         thread_ip_inc(p);
4649 }
4650
4651 static inline void
4652 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4653 {
4654         struct thread *t = &p->threads[p->thread_id];
4655         struct instruction *ip = t->ip;
4656
4657         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4658
4659         /* Structs. */
4660         ALU_HM(t, ip, -);
4661
4662         /* Thread. */
4663         thread_ip_inc(p);
4664 }
4665
4666 static inline void
4667 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4668 {
4669         struct thread *t = &p->threads[p->thread_id];
4670         struct instruction *ip = t->ip;
4671
4672         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4673
4674         /* Structs. */
4675         ALU_HH(t, ip, -);
4676
4677         /* Thread. */
4678         thread_ip_inc(p);
4679 }
4680
4681 static inline void
4682 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4683 {
4684         struct thread *t = &p->threads[p->thread_id];
4685         struct instruction *ip = t->ip;
4686
4687         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4688
4689         /* Structs. */
4690         ALU_MI(t, ip, -);
4691
4692         /* Thread. */
4693         thread_ip_inc(p);
4694 }
4695
4696 static inline void
4697 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4698 {
4699         struct thread *t = &p->threads[p->thread_id];
4700         struct instruction *ip = t->ip;
4701
4702         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4703
4704         /* Structs. */
4705         ALU_HI(t, ip, -);
4706
4707         /* Thread. */
4708         thread_ip_inc(p);
4709 }
4710
4711 static inline void
4712 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4713 {
4714         struct thread *t = &p->threads[p->thread_id];
4715         struct instruction *ip = t->ip;
4716
4717         TRACE("[Thread %2u] shl\n", p->thread_id);
4718
4719         /* Structs. */
4720         ALU(t, ip, <<);
4721
4722         /* Thread. */
4723         thread_ip_inc(p);
4724 }
4725
4726 static inline void
4727 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4728 {
4729         struct thread *t = &p->threads[p->thread_id];
4730         struct instruction *ip = t->ip;
4731
4732         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4733
4734         /* Structs. */
4735         ALU_MH(t, ip, <<);
4736
4737         /* Thread. */
4738         thread_ip_inc(p);
4739 }
4740
4741 static inline void
4742 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4743 {
4744         struct thread *t = &p->threads[p->thread_id];
4745         struct instruction *ip = t->ip;
4746
4747         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4748
4749         /* Structs. */
4750         ALU_HM(t, ip, <<);
4751
4752         /* Thread. */
4753         thread_ip_inc(p);
4754 }
4755
4756 static inline void
4757 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4758 {
4759         struct thread *t = &p->threads[p->thread_id];
4760         struct instruction *ip = t->ip;
4761
4762         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4763
4764         /* Structs. */
4765         ALU_HH(t, ip, <<);
4766
4767         /* Thread. */
4768         thread_ip_inc(p);
4769 }
4770
4771 static inline void
4772 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4773 {
4774         struct thread *t = &p->threads[p->thread_id];
4775         struct instruction *ip = t->ip;
4776
4777         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4778
4779         /* Structs. */
4780         ALU_MI(t, ip, <<);
4781
4782         /* Thread. */
4783         thread_ip_inc(p);
4784 }
4785
4786 static inline void
4787 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4788 {
4789         struct thread *t = &p->threads[p->thread_id];
4790         struct instruction *ip = t->ip;
4791
4792         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4793
4794         /* Structs. */
4795         ALU_HI(t, ip, <<);
4796
4797         /* Thread. */
4798         thread_ip_inc(p);
4799 }
4800
4801 static inline void
4802 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4803 {
4804         struct thread *t = &p->threads[p->thread_id];
4805         struct instruction *ip = t->ip;
4806
4807         TRACE("[Thread %2u] shr\n", p->thread_id);
4808
4809         /* Structs. */
4810         ALU(t, ip, >>);
4811
4812         /* Thread. */
4813         thread_ip_inc(p);
4814 }
4815
4816 static inline void
4817 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4818 {
4819         struct thread *t = &p->threads[p->thread_id];
4820         struct instruction *ip = t->ip;
4821
4822         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4823
4824         /* Structs. */
4825         ALU_MH(t, ip, >>);
4826
4827         /* Thread. */
4828         thread_ip_inc(p);
4829 }
4830
4831 static inline void
4832 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4833 {
4834         struct thread *t = &p->threads[p->thread_id];
4835         struct instruction *ip = t->ip;
4836
4837         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4838
4839         /* Structs. */
4840         ALU_HM(t, ip, >>);
4841
4842         /* Thread. */
4843         thread_ip_inc(p);
4844 }
4845
4846 static inline void
4847 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4848 {
4849         struct thread *t = &p->threads[p->thread_id];
4850         struct instruction *ip = t->ip;
4851
4852         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4853
4854         /* Structs. */
4855         ALU_HH(t, ip, >>);
4856
4857         /* Thread. */
4858         thread_ip_inc(p);
4859 }
4860
4861 static inline void
4862 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4863 {
4864         struct thread *t = &p->threads[p->thread_id];
4865         struct instruction *ip = t->ip;
4866
4867         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4868
4869         /* Structs. */
4870         ALU_MI(t, ip, >>);
4871
4872         /* Thread. */
4873         thread_ip_inc(p);
4874 }
4875
4876 static inline void
4877 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4878 {
4879         struct thread *t = &p->threads[p->thread_id];
4880         struct instruction *ip = t->ip;
4881
4882         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4883
4884         /* Structs. */
4885         ALU_HI(t, ip, >>);
4886
4887         /* Thread. */
4888         thread_ip_inc(p);
4889 }
4890
4891 static inline void
4892 instr_alu_and_exec(struct rte_swx_pipeline *p)
4893 {
4894         struct thread *t = &p->threads[p->thread_id];
4895         struct instruction *ip = t->ip;
4896
4897         TRACE("[Thread %2u] and\n", p->thread_id);
4898
4899         /* Structs. */
4900         ALU(t, ip, &);
4901
4902         /* Thread. */
4903         thread_ip_inc(p);
4904 }
4905
4906 static inline void
4907 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
4908 {
4909         struct thread *t = &p->threads[p->thread_id];
4910         struct instruction *ip = t->ip;
4911
4912         TRACE("[Thread %2u] and (mh)\n", p->thread_id);
4913
4914         /* Structs. */
4915         ALU_MH(t, ip, &);
4916
4917         /* Thread. */
4918         thread_ip_inc(p);
4919 }
4920
4921 static inline void
4922 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
4923 {
4924         struct thread *t = &p->threads[p->thread_id];
4925         struct instruction *ip = t->ip;
4926
4927         TRACE("[Thread %2u] and (hm)\n", p->thread_id);
4928
4929         /* Structs. */
4930         ALU_HM_FAST(t, ip, &);
4931
4932         /* Thread. */
4933         thread_ip_inc(p);
4934 }
4935
4936 static inline void
4937 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
4938 {
4939         struct thread *t = &p->threads[p->thread_id];
4940         struct instruction *ip = t->ip;
4941
4942         TRACE("[Thread %2u] and (hh)\n", p->thread_id);
4943
4944         /* Structs. */
4945         ALU_HH_FAST(t, ip, &);
4946
4947         /* Thread. */
4948         thread_ip_inc(p);
4949 }
4950
4951 static inline void
4952 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4953 {
4954         struct thread *t = &p->threads[p->thread_id];
4955         struct instruction *ip = t->ip;
4956
4957         TRACE("[Thread %2u] and (i)\n", p->thread_id);
4958
4959         /* Structs. */
4960         ALU_I(t, ip, &);
4961
4962         /* Thread. */
4963         thread_ip_inc(p);
4964 }
4965
4966 static inline void
4967 instr_alu_or_exec(struct rte_swx_pipeline *p)
4968 {
4969         struct thread *t = &p->threads[p->thread_id];
4970         struct instruction *ip = t->ip;
4971
4972         TRACE("[Thread %2u] or\n", p->thread_id);
4973
4974         /* Structs. */
4975         ALU(t, ip, |);
4976
4977         /* Thread. */
4978         thread_ip_inc(p);
4979 }
4980
4981 static inline void
4982 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
4983 {
4984         struct thread *t = &p->threads[p->thread_id];
4985         struct instruction *ip = t->ip;
4986
4987         TRACE("[Thread %2u] or (mh)\n", p->thread_id);
4988
4989         /* Structs. */
4990         ALU_MH(t, ip, |);
4991
4992         /* Thread. */
4993         thread_ip_inc(p);
4994 }
4995
4996 static inline void
4997 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
4998 {
4999         struct thread *t = &p->threads[p->thread_id];
5000         struct instruction *ip = t->ip;
5001
5002         TRACE("[Thread %2u] or (hm)\n", p->thread_id);
5003
5004         /* Structs. */
5005         ALU_HM_FAST(t, ip, |);
5006
5007         /* Thread. */
5008         thread_ip_inc(p);
5009 }
5010
5011 static inline void
5012 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
5013 {
5014         struct thread *t = &p->threads[p->thread_id];
5015         struct instruction *ip = t->ip;
5016
5017         TRACE("[Thread %2u] or (hh)\n", p->thread_id);
5018
5019         /* Structs. */
5020         ALU_HH_FAST(t, ip, |);
5021
5022         /* Thread. */
5023         thread_ip_inc(p);
5024 }
5025
5026 static inline void
5027 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
5028 {
5029         struct thread *t = &p->threads[p->thread_id];
5030         struct instruction *ip = t->ip;
5031
5032         TRACE("[Thread %2u] or (i)\n", p->thread_id);
5033
5034         /* Structs. */
5035         ALU_I(t, ip, |);
5036
5037         /* Thread. */
5038         thread_ip_inc(p);
5039 }
5040
5041 static inline void
5042 instr_alu_xor_exec(struct rte_swx_pipeline *p)
5043 {
5044         struct thread *t = &p->threads[p->thread_id];
5045         struct instruction *ip = t->ip;
5046
5047         TRACE("[Thread %2u] xor\n", p->thread_id);
5048
5049         /* Structs. */
5050         ALU(t, ip, ^);
5051
5052         /* Thread. */
5053         thread_ip_inc(p);
5054 }
5055
5056 static inline void
5057 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
5058 {
5059         struct thread *t = &p->threads[p->thread_id];
5060         struct instruction *ip = t->ip;
5061
5062         TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
5063
5064         /* Structs. */
5065         ALU_MH(t, ip, ^);
5066
5067         /* Thread. */
5068         thread_ip_inc(p);
5069 }
5070
5071 static inline void
5072 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
5073 {
5074         struct thread *t = &p->threads[p->thread_id];
5075         struct instruction *ip = t->ip;
5076
5077         TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
5078
5079         /* Structs. */
5080         ALU_HM_FAST(t, ip, ^);
5081
5082         /* Thread. */
5083         thread_ip_inc(p);
5084 }
5085
5086 static inline void
5087 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
5088 {
5089         struct thread *t = &p->threads[p->thread_id];
5090         struct instruction *ip = t->ip;
5091
5092         TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
5093
5094         /* Structs. */
5095         ALU_HH_FAST(t, ip, ^);
5096
5097         /* Thread. */
5098         thread_ip_inc(p);
5099 }
5100
5101 static inline void
5102 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
5103 {
5104         struct thread *t = &p->threads[p->thread_id];
5105         struct instruction *ip = t->ip;
5106
5107         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
5108
5109         /* Structs. */
5110         ALU_I(t, ip, ^);
5111
5112         /* Thread. */
5113         thread_ip_inc(p);
5114 }
5115
5116 static inline void
5117 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
5118 {
5119         struct thread *t = &p->threads[p->thread_id];
5120         struct instruction *ip = t->ip;
5121         uint8_t *dst_struct, *src_struct;
5122         uint16_t *dst16_ptr, dst;
5123         uint64_t *src64_ptr, src64, src64_mask, src;
5124         uint64_t r;
5125
5126         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
5127
5128         /* Structs. */
5129         dst_struct = t->structs[ip->alu.dst.struct_id];
5130         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5131         dst = *dst16_ptr;
5132
5133         src_struct = t->structs[ip->alu.src.struct_id];
5134         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
5135         src64 = *src64_ptr;
5136         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
5137         src = src64 & src64_mask;
5138
5139         r = dst;
5140         r = ~r & 0xFFFF;
5141
5142         /* The first input (r) is a 16-bit number. The second and the third
5143          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
5144          * three numbers (output r) is a 34-bit number.
5145          */
5146         r += (src >> 32) + (src & 0xFFFFFFFF);
5147
5148         /* The first input is a 16-bit number. The second input is an 18-bit
5149          * number. In the worst case scenario, the sum of the two numbers is a
5150          * 19-bit number.
5151          */
5152         r = (r & 0xFFFF) + (r >> 16);
5153
5154         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5155          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
5156          */
5157         r = (r & 0xFFFF) + (r >> 16);
5158
5159         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5160          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5161          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
5162          * therefore the output r is always a 16-bit number.
5163          */
5164         r = (r & 0xFFFF) + (r >> 16);
5165
5166         r = ~r & 0xFFFF;
5167         r = r ? r : 0xFFFF;
5168
5169         *dst16_ptr = (uint16_t)r;
5170
5171         /* Thread. */
5172         thread_ip_inc(p);
5173 }
5174
5175 static inline void
5176 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
5177 {
5178         struct thread *t = &p->threads[p->thread_id];
5179         struct instruction *ip = t->ip;
5180         uint8_t *dst_struct, *src_struct;
5181         uint16_t *dst16_ptr, dst;
5182         uint64_t *src64_ptr, src64, src64_mask, src;
5183         uint64_t r;
5184
5185         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
5186
5187         /* Structs. */
5188         dst_struct = t->structs[ip->alu.dst.struct_id];
5189         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5190         dst = *dst16_ptr;
5191
5192         src_struct = t->structs[ip->alu.src.struct_id];
5193         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
5194         src64 = *src64_ptr;
5195         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
5196         src = src64 & src64_mask;
5197
5198         r = dst;
5199         r = ~r & 0xFFFF;
5200
5201         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
5202          * the following sequence of operations in 2's complement arithmetic:
5203          *    a '- b = (a - b) % 0xFFFF.
5204          *
5205          * In order to prevent an underflow for the below subtraction, in which
5206          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
5207          * minuend), we first add a multiple of the 0xFFFF modulus to the
5208          * minuend. The number we add to the minuend needs to be a 34-bit number
5209          * or higher, so for readability reasons we picked the 36-bit multiple.
5210          * We are effectively turning the 16-bit minuend into a 36-bit number:
5211          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
5212          */
5213         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
5214
5215         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
5216          * result (the output r) is a 36-bit number.
5217          */
5218         r -= (src >> 32) + (src & 0xFFFFFFFF);
5219
5220         /* The first input is a 16-bit number. The second input is a 20-bit
5221          * number. Their sum is a 21-bit number.
5222          */
5223         r = (r & 0xFFFF) + (r >> 16);
5224
5225         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5226          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
5227          */
5228         r = (r & 0xFFFF) + (r >> 16);
5229
5230         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5231          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5232          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5233          * generated, therefore the output r is always a 16-bit number.
5234          */
5235         r = (r & 0xFFFF) + (r >> 16);
5236
5237         r = ~r & 0xFFFF;
5238         r = r ? r : 0xFFFF;
5239
5240         *dst16_ptr = (uint16_t)r;
5241
5242         /* Thread. */
5243         thread_ip_inc(p);
5244 }
5245
5246 static inline void
5247 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
5248 {
5249         struct thread *t = &p->threads[p->thread_id];
5250         struct instruction *ip = t->ip;
5251         uint8_t *dst_struct, *src_struct;
5252         uint16_t *dst16_ptr;
5253         uint32_t *src32_ptr;
5254         uint64_t r0, r1;
5255
5256         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
5257
5258         /* Structs. */
5259         dst_struct = t->structs[ip->alu.dst.struct_id];
5260         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5261
5262         src_struct = t->structs[ip->alu.src.struct_id];
5263         src32_ptr = (uint32_t *)&src_struct[0];
5264
5265         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
5266         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
5267         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
5268         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
5269         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
5270
5271         /* The first input is a 16-bit number. The second input is a 19-bit
5272          * number. Their sum is a 20-bit number.
5273          */
5274         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5275
5276         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5277          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
5278          */
5279         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5280
5281         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5282          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5283          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
5284          * generated, therefore the output r is always a 16-bit number.
5285          */
5286         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5287
5288         r0 = ~r0 & 0xFFFF;
5289         r0 = r0 ? r0 : 0xFFFF;
5290
5291         *dst16_ptr = (uint16_t)r0;
5292
5293         /* Thread. */
5294         thread_ip_inc(p);
5295 }
5296
5297 static inline void
5298 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
5299 {
5300         struct thread *t = &p->threads[p->thread_id];
5301         struct instruction *ip = t->ip;
5302         uint8_t *dst_struct, *src_struct;
5303         uint16_t *dst16_ptr;
5304         uint32_t *src32_ptr;
5305         uint64_t r = 0;
5306         uint32_t i;
5307
5308         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
5309
5310         /* Structs. */
5311         dst_struct = t->structs[ip->alu.dst.struct_id];
5312         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5313
5314         src_struct = t->structs[ip->alu.src.struct_id];
5315         src32_ptr = (uint32_t *)&src_struct[0];
5316
5317         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
5318          * Therefore, in the worst case scenario, a 35-bit number is added to a
5319          * 16-bit number (the input r), so the output r is 36-bit number.
5320          */
5321         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
5322                 r += *src32_ptr;
5323
5324         /* The first input is a 16-bit number. The second input is a 20-bit
5325          * number. Their sum is a 21-bit number.
5326          */
5327         r = (r & 0xFFFF) + (r >> 16);
5328
5329         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5330          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
5331          */
5332         r = (r & 0xFFFF) + (r >> 16);
5333
5334         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5335          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5336          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5337          * generated, therefore the output r is always a 16-bit number.
5338          */
5339         r = (r & 0xFFFF) + (r >> 16);
5340
5341         r = ~r & 0xFFFF;
5342         r = r ? r : 0xFFFF;
5343
5344         *dst16_ptr = (uint16_t)r;
5345
5346         /* Thread. */
5347         thread_ip_inc(p);
5348 }
5349
5350 /*
5351  * Register array.
5352  */
5353 static struct regarray *
5354 regarray_find(struct rte_swx_pipeline *p, const char *name);
5355
5356 static int
5357 instr_regprefetch_translate(struct rte_swx_pipeline *p,
5358                       struct action *action,
5359                       char **tokens,
5360                       int n_tokens,
5361                       struct instruction *instr,
5362                       struct instruction_data *data __rte_unused)
5363 {
5364         char *regarray = tokens[1], *idx = tokens[2];
5365         struct regarray *r;
5366         struct field *fidx;
5367         uint32_t idx_struct_id, idx_val;
5368
5369         CHECK(n_tokens == 3, EINVAL);
5370
5371         r = regarray_find(p, regarray);
5372         CHECK(r, EINVAL);
5373
5374         /* REGPREFETCH_RH, REGPREFETCH_RM. */
5375         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5376         if (fidx) {
5377                 CHECK(!fidx->var_size, EINVAL);
5378
5379                 instr->type = INSTR_REGPREFETCH_RM;
5380                 if (idx[0] == 'h')
5381                         instr->type = INSTR_REGPREFETCH_RH;
5382
5383                 instr->regarray.regarray_id = r->id;
5384                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5385                 instr->regarray.idx.n_bits = fidx->n_bits;
5386                 instr->regarray.idx.offset = fidx->offset / 8;
5387                 instr->regarray.dstsrc_val = 0; /* Unused. */
5388                 return 0;
5389         }
5390
5391         /* REGPREFETCH_RI. */
5392         idx_val = strtoul(idx, &idx, 0);
5393         CHECK(!idx[0], EINVAL);
5394
5395         instr->type = INSTR_REGPREFETCH_RI;
5396         instr->regarray.regarray_id = r->id;
5397         instr->regarray.idx_val = idx_val;
5398         instr->regarray.dstsrc_val = 0; /* Unused. */
5399         return 0;
5400 }
5401
5402 static int
5403 instr_regrd_translate(struct rte_swx_pipeline *p,
5404                       struct action *action,
5405                       char **tokens,
5406                       int n_tokens,
5407                       struct instruction *instr,
5408                       struct instruction_data *data __rte_unused)
5409 {
5410         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
5411         struct regarray *r;
5412         struct field *fdst, *fidx;
5413         uint32_t dst_struct_id, idx_struct_id, idx_val;
5414
5415         CHECK(n_tokens == 4, EINVAL);
5416
5417         r = regarray_find(p, regarray);
5418         CHECK(r, EINVAL);
5419
5420         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
5421         CHECK(fdst, EINVAL);
5422         CHECK(!fdst->var_size, EINVAL);
5423
5424         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
5425         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5426         if (fidx) {
5427                 CHECK(!fidx->var_size, EINVAL);
5428
5429                 instr->type = INSTR_REGRD_MRM;
5430                 if (dst[0] == 'h' && idx[0] != 'h')
5431                         instr->type = INSTR_REGRD_HRM;
5432                 if (dst[0] != 'h' && idx[0] == 'h')
5433                         instr->type = INSTR_REGRD_MRH;
5434                 if (dst[0] == 'h' && idx[0] == 'h')
5435                         instr->type = INSTR_REGRD_HRH;
5436
5437                 instr->regarray.regarray_id = r->id;
5438                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5439                 instr->regarray.idx.n_bits = fidx->n_bits;
5440                 instr->regarray.idx.offset = fidx->offset / 8;
5441                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5442                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
5443                 instr->regarray.dstsrc.offset = fdst->offset / 8;
5444                 return 0;
5445         }
5446
5447         /* REGRD_MRI, REGRD_HRI. */
5448         idx_val = strtoul(idx, &idx, 0);
5449         CHECK(!idx[0], EINVAL);
5450
5451         instr->type = INSTR_REGRD_MRI;
5452         if (dst[0] == 'h')
5453                 instr->type = INSTR_REGRD_HRI;
5454
5455         instr->regarray.regarray_id = r->id;
5456         instr->regarray.idx_val = idx_val;
5457         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5458         instr->regarray.dstsrc.n_bits = fdst->n_bits;
5459         instr->regarray.dstsrc.offset = fdst->offset / 8;
5460         return 0;
5461 }
5462
5463 static int
5464 instr_regwr_translate(struct rte_swx_pipeline *p,
5465                       struct action *action,
5466                       char **tokens,
5467                       int n_tokens,
5468                       struct instruction *instr,
5469                       struct instruction_data *data __rte_unused)
5470 {
5471         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5472         struct regarray *r;
5473         struct field *fidx, *fsrc;
5474         uint64_t src_val;
5475         uint32_t idx_struct_id, idx_val, src_struct_id;
5476
5477         CHECK(n_tokens == 4, EINVAL);
5478
5479         r = regarray_find(p, regarray);
5480         CHECK(r, EINVAL);
5481
5482         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
5483         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5484         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5485         if (fidx && fsrc) {
5486                 CHECK(!fidx->var_size, EINVAL);
5487                 CHECK(!fsrc->var_size, EINVAL);
5488
5489                 instr->type = INSTR_REGWR_RMM;
5490                 if (idx[0] == 'h' && src[0] != 'h')
5491                         instr->type = INSTR_REGWR_RHM;
5492                 if (idx[0] != 'h' && src[0] == 'h')
5493                         instr->type = INSTR_REGWR_RMH;
5494                 if (idx[0] == 'h' && src[0] == 'h')
5495                         instr->type = INSTR_REGWR_RHH;
5496
5497                 instr->regarray.regarray_id = r->id;
5498                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5499                 instr->regarray.idx.n_bits = fidx->n_bits;
5500                 instr->regarray.idx.offset = fidx->offset / 8;
5501                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5502                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5503                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5504                 return 0;
5505         }
5506
5507         /* REGWR_RHI, REGWR_RMI. */
5508         if (fidx && !fsrc) {
5509                 CHECK(!fidx->var_size, EINVAL);
5510
5511                 src_val = strtoull(src, &src, 0);
5512                 CHECK(!src[0], EINVAL);
5513
5514                 instr->type = INSTR_REGWR_RMI;
5515                 if (idx[0] == 'h')
5516                         instr->type = INSTR_REGWR_RHI;
5517
5518                 instr->regarray.regarray_id = r->id;
5519                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5520                 instr->regarray.idx.n_bits = fidx->n_bits;
5521                 instr->regarray.idx.offset = fidx->offset / 8;
5522                 instr->regarray.dstsrc_val = src_val;
5523                 return 0;
5524         }
5525
5526         /* REGWR_RIH, REGWR_RIM. */
5527         if (!fidx && fsrc) {
5528                 idx_val = strtoul(idx, &idx, 0);
5529                 CHECK(!idx[0], EINVAL);
5530
5531                 CHECK(!fsrc->var_size, EINVAL);
5532
5533                 instr->type = INSTR_REGWR_RIM;
5534                 if (src[0] == 'h')
5535                         instr->type = INSTR_REGWR_RIH;
5536
5537                 instr->regarray.regarray_id = r->id;
5538                 instr->regarray.idx_val = idx_val;
5539                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5540                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5541                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5542                 return 0;
5543         }
5544
5545         /* REGWR_RII. */
5546         src_val = strtoull(src, &src, 0);
5547         CHECK(!src[0], EINVAL);
5548
5549         idx_val = strtoul(idx, &idx, 0);
5550         CHECK(!idx[0], EINVAL);
5551
5552         instr->type = INSTR_REGWR_RII;
5553         instr->regarray.idx_val = idx_val;
5554         instr->regarray.dstsrc_val = src_val;
5555
5556         return 0;
5557 }
5558
5559 static int
5560 instr_regadd_translate(struct rte_swx_pipeline *p,
5561                        struct action *action,
5562                        char **tokens,
5563                        int n_tokens,
5564                        struct instruction *instr,
5565                        struct instruction_data *data __rte_unused)
5566 {
5567         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5568         struct regarray *r;
5569         struct field *fidx, *fsrc;
5570         uint64_t src_val;
5571         uint32_t idx_struct_id, idx_val, src_struct_id;
5572
5573         CHECK(n_tokens == 4, EINVAL);
5574
5575         r = regarray_find(p, regarray);
5576         CHECK(r, EINVAL);
5577
5578         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
5579         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5580         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5581         if (fidx && fsrc) {
5582                 CHECK(!fidx->var_size, EINVAL);
5583                 CHECK(!fsrc->var_size, EINVAL);
5584
5585                 instr->type = INSTR_REGADD_RMM;
5586                 if (idx[0] == 'h' && src[0] != 'h')
5587                         instr->type = INSTR_REGADD_RHM;
5588                 if (idx[0] != 'h' && src[0] == 'h')
5589                         instr->type = INSTR_REGADD_RMH;
5590                 if (idx[0] == 'h' && src[0] == 'h')
5591                         instr->type = INSTR_REGADD_RHH;
5592
5593                 instr->regarray.regarray_id = r->id;
5594                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5595                 instr->regarray.idx.n_bits = fidx->n_bits;
5596                 instr->regarray.idx.offset = fidx->offset / 8;
5597                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5598                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5599                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5600                 return 0;
5601         }
5602
5603         /* REGADD_RHI, REGADD_RMI. */
5604         if (fidx && !fsrc) {
5605                 CHECK(!fidx->var_size, EINVAL);
5606
5607                 src_val = strtoull(src, &src, 0);
5608                 CHECK(!src[0], EINVAL);
5609
5610                 instr->type = INSTR_REGADD_RMI;
5611                 if (idx[0] == 'h')
5612                         instr->type = INSTR_REGADD_RHI;
5613
5614                 instr->regarray.regarray_id = r->id;
5615                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5616                 instr->regarray.idx.n_bits = fidx->n_bits;
5617                 instr->regarray.idx.offset = fidx->offset / 8;
5618                 instr->regarray.dstsrc_val = src_val;
5619                 return 0;
5620         }
5621
5622         /* REGADD_RIH, REGADD_RIM. */
5623         if (!fidx && fsrc) {
5624                 idx_val = strtoul(idx, &idx, 0);
5625                 CHECK(!idx[0], EINVAL);
5626
5627                 CHECK(!fsrc->var_size, EINVAL);
5628
5629                 instr->type = INSTR_REGADD_RIM;
5630                 if (src[0] == 'h')
5631                         instr->type = INSTR_REGADD_RIH;
5632
5633                 instr->regarray.regarray_id = r->id;
5634                 instr->regarray.idx_val = idx_val;
5635                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5636                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5637                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5638                 return 0;
5639         }
5640
5641         /* REGADD_RII. */
5642         src_val = strtoull(src, &src, 0);
5643         CHECK(!src[0], EINVAL);
5644
5645         idx_val = strtoul(idx, &idx, 0);
5646         CHECK(!idx[0], EINVAL);
5647
5648         instr->type = INSTR_REGADD_RII;
5649         instr->regarray.idx_val = idx_val;
5650         instr->regarray.dstsrc_val = src_val;
5651         return 0;
5652 }
5653
5654 static inline uint64_t *
5655 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
5656 {
5657         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5658         return r->regarray;
5659 }
5660
5661 static inline uint64_t
5662 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5663 {
5664         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5665
5666         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5667         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5668         uint64_t idx64 = *idx64_ptr;
5669         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
5670         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5671
5672         return idx;
5673 }
5674
5675 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5676
5677 static inline uint64_t
5678 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5679 {
5680         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5681
5682         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5683         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5684         uint64_t idx64 = *idx64_ptr;
5685         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
5686
5687         return idx;
5688 }
5689
5690 #else
5691
5692 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
5693
5694 #endif
5695
5696 static inline uint64_t
5697 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5698 {
5699         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5700
5701         uint64_t idx = ip->regarray.idx_val & r->size_mask;
5702
5703         return idx;
5704 }
5705
5706 static inline uint64_t
5707 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
5708 {
5709         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5710         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5711         uint64_t src64 = *src64_ptr;
5712         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5713         uint64_t src = src64 & src64_mask;
5714
5715         return src;
5716 }
5717
5718 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5719
5720 static inline uint64_t
5721 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
5722 {
5723         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5724         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5725         uint64_t src64 = *src64_ptr;
5726         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
5727
5728         return src;
5729 }
5730
5731 #else
5732
5733 #define instr_regarray_src_nbo instr_regarray_src_hbo
5734
5735 #endif
5736
5737 static inline void
5738 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5739 {
5740         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5741         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5742         uint64_t dst64 = *dst64_ptr;
5743         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5744
5745         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5746
5747 }
5748
5749 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5750
5751 static inline void
5752 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5753 {
5754         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5755         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5756         uint64_t dst64 = *dst64_ptr;
5757         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5758
5759         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
5760         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5761 }
5762
5763 #else
5764
5765 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
5766
5767 #endif
5768
5769 static inline void
5770 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
5771 {
5772         struct thread *t = &p->threads[p->thread_id];
5773         struct instruction *ip = t->ip;
5774         uint64_t *regarray, idx;
5775
5776         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
5777
5778         /* Structs. */
5779         regarray = instr_regarray_regarray(p, ip);
5780         idx = instr_regarray_idx_nbo(p, t, ip);
5781         rte_prefetch0(&regarray[idx]);
5782
5783         /* Thread. */
5784         thread_ip_inc(p);
5785 }
5786
5787 static inline void
5788 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
5789 {
5790         struct thread *t = &p->threads[p->thread_id];
5791         struct instruction *ip = t->ip;
5792         uint64_t *regarray, idx;
5793
5794         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
5795
5796         /* Structs. */
5797         regarray = instr_regarray_regarray(p, ip);
5798         idx = instr_regarray_idx_hbo(p, t, ip);
5799         rte_prefetch0(&regarray[idx]);
5800
5801         /* Thread. */
5802         thread_ip_inc(p);
5803 }
5804
5805 static inline void
5806 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
5807 {
5808         struct thread *t = &p->threads[p->thread_id];
5809         struct instruction *ip = t->ip;
5810         uint64_t *regarray, idx;
5811
5812         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
5813
5814         /* Structs. */
5815         regarray = instr_regarray_regarray(p, ip);
5816         idx = instr_regarray_idx_imm(p, ip);
5817         rte_prefetch0(&regarray[idx]);
5818
5819         /* Thread. */
5820         thread_ip_inc(p);
5821 }
5822
5823 static inline void
5824 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
5825 {
5826         struct thread *t = &p->threads[p->thread_id];
5827         struct instruction *ip = t->ip;
5828         uint64_t *regarray, idx;
5829
5830         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
5831
5832         /* Structs. */
5833         regarray = instr_regarray_regarray(p, ip);
5834         idx = instr_regarray_idx_nbo(p, t, ip);
5835         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5836
5837         /* Thread. */
5838         thread_ip_inc(p);
5839 }
5840
5841 static inline void
5842 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
5843 {
5844         struct thread *t = &p->threads[p->thread_id];
5845         struct instruction *ip = t->ip;
5846         uint64_t *regarray, idx;
5847
5848         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
5849
5850         /* Structs. */
5851         regarray = instr_regarray_regarray(p, ip);
5852         idx = instr_regarray_idx_hbo(p, t, ip);
5853         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5854
5855         /* Thread. */
5856         thread_ip_inc(p);
5857 }
5858
5859 static inline void
5860 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
5861 {
5862         struct thread *t = &p->threads[p->thread_id];
5863         struct instruction *ip = t->ip;
5864         uint64_t *regarray, idx;
5865
5866         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
5867
5868         /* Structs. */
5869         regarray = instr_regarray_regarray(p, ip);
5870         idx = instr_regarray_idx_nbo(p, t, ip);
5871         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5872
5873         /* Thread. */
5874         thread_ip_inc(p);
5875 }
5876
5877 static inline void
5878 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
5879 {
5880         struct thread *t = &p->threads[p->thread_id];
5881         struct instruction *ip = t->ip;
5882         uint64_t *regarray, idx;
5883
5884         /* Structs. */
5885         regarray = instr_regarray_regarray(p, ip);
5886         idx = instr_regarray_idx_hbo(p, t, ip);
5887         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5888
5889         /* Thread. */
5890         thread_ip_inc(p);
5891 }
5892
5893 static inline void
5894 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
5895 {
5896         struct thread *t = &p->threads[p->thread_id];
5897         struct instruction *ip = t->ip;
5898         uint64_t *regarray, idx;
5899
5900         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
5901
5902         /* Structs. */
5903         regarray = instr_regarray_regarray(p, ip);
5904         idx = instr_regarray_idx_imm(p, ip);
5905         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5906
5907         /* Thread. */
5908         thread_ip_inc(p);
5909 }
5910
5911 static inline void
5912 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
5913 {
5914         struct thread *t = &p->threads[p->thread_id];
5915         struct instruction *ip = t->ip;
5916         uint64_t *regarray, idx;
5917
5918         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
5919
5920         /* Structs. */
5921         regarray = instr_regarray_regarray(p, ip);
5922         idx = instr_regarray_idx_imm(p, ip);
5923         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5924
5925         /* Thread. */
5926         thread_ip_inc(p);
5927 }
5928
5929 static inline void
5930 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
5931 {
5932         struct thread *t = &p->threads[p->thread_id];
5933         struct instruction *ip = t->ip;
5934         uint64_t *regarray, idx, src;
5935
5936         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
5937
5938         /* Structs. */
5939         regarray = instr_regarray_regarray(p, ip);
5940         idx = instr_regarray_idx_nbo(p, t, ip);
5941         src = instr_regarray_src_nbo(t, ip);
5942         regarray[idx] = src;
5943
5944         /* Thread. */
5945         thread_ip_inc(p);
5946 }
5947
5948 static inline void
5949 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
5950 {
5951         struct thread *t = &p->threads[p->thread_id];
5952         struct instruction *ip = t->ip;
5953         uint64_t *regarray, idx, src;
5954
5955         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
5956
5957         /* Structs. */
5958         regarray = instr_regarray_regarray(p, ip);
5959         idx = instr_regarray_idx_nbo(p, t, ip);
5960         src = instr_regarray_src_hbo(t, ip);
5961         regarray[idx] = src;
5962
5963         /* Thread. */
5964         thread_ip_inc(p);
5965 }
5966
5967 static inline void
5968 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
5969 {
5970         struct thread *t = &p->threads[p->thread_id];
5971         struct instruction *ip = t->ip;
5972         uint64_t *regarray, idx, src;
5973
5974         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
5975
5976         /* Structs. */
5977         regarray = instr_regarray_regarray(p, ip);
5978         idx = instr_regarray_idx_hbo(p, t, ip);
5979         src = instr_regarray_src_nbo(t, ip);
5980         regarray[idx] = src;
5981
5982         /* Thread. */
5983         thread_ip_inc(p);
5984 }
5985
5986 static inline void
5987 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
5988 {
5989         struct thread *t = &p->threads[p->thread_id];
5990         struct instruction *ip = t->ip;
5991         uint64_t *regarray, idx, src;
5992
5993         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
5994
5995         /* Structs. */
5996         regarray = instr_regarray_regarray(p, ip);
5997         idx = instr_regarray_idx_hbo(p, t, ip);
5998         src = instr_regarray_src_hbo(t, ip);
5999         regarray[idx] = src;
6000
6001         /* Thread. */
6002         thread_ip_inc(p);
6003 }
6004
6005 static inline void
6006 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
6007 {
6008         struct thread *t = &p->threads[p->thread_id];
6009         struct instruction *ip = t->ip;
6010         uint64_t *regarray, idx, src;
6011
6012         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
6013
6014         /* Structs. */
6015         regarray = instr_regarray_regarray(p, ip);
6016         idx = instr_regarray_idx_nbo(p, t, ip);
6017         src = ip->regarray.dstsrc_val;
6018         regarray[idx] = src;
6019
6020         /* Thread. */
6021         thread_ip_inc(p);
6022 }
6023
6024 static inline void
6025 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
6026 {
6027         struct thread *t = &p->threads[p->thread_id];
6028         struct instruction *ip = t->ip;
6029         uint64_t *regarray, idx, src;
6030
6031         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
6032
6033         /* Structs. */
6034         regarray = instr_regarray_regarray(p, ip);
6035         idx = instr_regarray_idx_hbo(p, t, ip);
6036         src = ip->regarray.dstsrc_val;
6037         regarray[idx] = src;
6038
6039         /* Thread. */
6040         thread_ip_inc(p);
6041 }
6042
6043 static inline void
6044 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
6045 {
6046         struct thread *t = &p->threads[p->thread_id];
6047         struct instruction *ip = t->ip;
6048         uint64_t *regarray, idx, src;
6049
6050         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
6051
6052         /* Structs. */
6053         regarray = instr_regarray_regarray(p, ip);
6054         idx = instr_regarray_idx_imm(p, ip);
6055         src = instr_regarray_src_nbo(t, ip);
6056         regarray[idx] = src;
6057
6058         /* Thread. */
6059         thread_ip_inc(p);
6060 }
6061
6062 static inline void
6063 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
6064 {
6065         struct thread *t = &p->threads[p->thread_id];
6066         struct instruction *ip = t->ip;
6067         uint64_t *regarray, idx, src;
6068
6069         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
6070
6071         /* Structs. */
6072         regarray = instr_regarray_regarray(p, ip);
6073         idx = instr_regarray_idx_imm(p, ip);
6074         src = instr_regarray_src_hbo(t, ip);
6075         regarray[idx] = src;
6076
6077         /* Thread. */
6078         thread_ip_inc(p);
6079 }
6080
6081 static inline void
6082 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
6083 {
6084         struct thread *t = &p->threads[p->thread_id];
6085         struct instruction *ip = t->ip;
6086         uint64_t *regarray, idx, src;
6087
6088         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
6089
6090         /* Structs. */
6091         regarray = instr_regarray_regarray(p, ip);
6092         idx = instr_regarray_idx_imm(p, ip);
6093         src = ip->regarray.dstsrc_val;
6094         regarray[idx] = src;
6095
6096         /* Thread. */
6097         thread_ip_inc(p);
6098 }
6099
6100 static inline void
6101 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
6102 {
6103         struct thread *t = &p->threads[p->thread_id];
6104         struct instruction *ip = t->ip;
6105         uint64_t *regarray, idx, src;
6106
6107         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
6108
6109         /* Structs. */
6110         regarray = instr_regarray_regarray(p, ip);
6111         idx = instr_regarray_idx_nbo(p, t, ip);
6112         src = instr_regarray_src_nbo(t, ip);
6113         regarray[idx] += src;
6114
6115         /* Thread. */
6116         thread_ip_inc(p);
6117 }
6118
6119 static inline void
6120 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
6121 {
6122         struct thread *t = &p->threads[p->thread_id];
6123         struct instruction *ip = t->ip;
6124         uint64_t *regarray, idx, src;
6125
6126         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
6127
6128         /* Structs. */
6129         regarray = instr_regarray_regarray(p, ip);
6130         idx = instr_regarray_idx_nbo(p, t, ip);
6131         src = instr_regarray_src_hbo(t, ip);
6132         regarray[idx] += src;
6133
6134         /* Thread. */
6135         thread_ip_inc(p);
6136 }
6137
6138 static inline void
6139 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
6140 {
6141         struct thread *t = &p->threads[p->thread_id];
6142         struct instruction *ip = t->ip;
6143         uint64_t *regarray, idx, src;
6144
6145         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
6146
6147         /* Structs. */
6148         regarray = instr_regarray_regarray(p, ip);
6149         idx = instr_regarray_idx_hbo(p, t, ip);
6150         src = instr_regarray_src_nbo(t, ip);
6151         regarray[idx] += src;
6152
6153         /* Thread. */
6154         thread_ip_inc(p);
6155 }
6156
6157 static inline void
6158 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
6159 {
6160         struct thread *t = &p->threads[p->thread_id];
6161         struct instruction *ip = t->ip;
6162         uint64_t *regarray, idx, src;
6163
6164         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
6165
6166         /* Structs. */
6167         regarray = instr_regarray_regarray(p, ip);
6168         idx = instr_regarray_idx_hbo(p, t, ip);
6169         src = instr_regarray_src_hbo(t, ip);
6170         regarray[idx] += src;
6171
6172         /* Thread. */
6173         thread_ip_inc(p);
6174 }
6175
6176 static inline void
6177 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
6178 {
6179         struct thread *t = &p->threads[p->thread_id];
6180         struct instruction *ip = t->ip;
6181         uint64_t *regarray, idx, src;
6182
6183         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
6184
6185         /* Structs. */
6186         regarray = instr_regarray_regarray(p, ip);
6187         idx = instr_regarray_idx_nbo(p, t, ip);
6188         src = ip->regarray.dstsrc_val;
6189         regarray[idx] += src;
6190
6191         /* Thread. */
6192         thread_ip_inc(p);
6193 }
6194
6195 static inline void
6196 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
6197 {
6198         struct thread *t = &p->threads[p->thread_id];
6199         struct instruction *ip = t->ip;
6200         uint64_t *regarray, idx, src;
6201
6202         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
6203
6204         /* Structs. */
6205         regarray = instr_regarray_regarray(p, ip);
6206         idx = instr_regarray_idx_hbo(p, t, ip);
6207         src = ip->regarray.dstsrc_val;
6208         regarray[idx] += src;
6209
6210         /* Thread. */
6211         thread_ip_inc(p);
6212 }
6213
6214 static inline void
6215 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
6216 {
6217         struct thread *t = &p->threads[p->thread_id];
6218         struct instruction *ip = t->ip;
6219         uint64_t *regarray, idx, src;
6220
6221         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
6222
6223         /* Structs. */
6224         regarray = instr_regarray_regarray(p, ip);
6225         idx = instr_regarray_idx_imm(p, ip);
6226         src = instr_regarray_src_nbo(t, ip);
6227         regarray[idx] += src;
6228
6229         /* Thread. */
6230         thread_ip_inc(p);
6231 }
6232
6233 static inline void
6234 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
6235 {
6236         struct thread *t = &p->threads[p->thread_id];
6237         struct instruction *ip = t->ip;
6238         uint64_t *regarray, idx, src;
6239
6240         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
6241
6242         /* Structs. */
6243         regarray = instr_regarray_regarray(p, ip);
6244         idx = instr_regarray_idx_imm(p, ip);
6245         src = instr_regarray_src_hbo(t, ip);
6246         regarray[idx] += src;
6247
6248         /* Thread. */
6249         thread_ip_inc(p);
6250 }
6251
6252 static inline void
6253 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
6254 {
6255         struct thread *t = &p->threads[p->thread_id];
6256         struct instruction *ip = t->ip;
6257         uint64_t *regarray, idx, src;
6258
6259         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
6260
6261         /* Structs. */
6262         regarray = instr_regarray_regarray(p, ip);
6263         idx = instr_regarray_idx_imm(p, ip);
6264         src = ip->regarray.dstsrc_val;
6265         regarray[idx] += src;
6266
6267         /* Thread. */
6268         thread_ip_inc(p);
6269 }
6270
6271 /*
6272  * metarray.
6273  */
6274 static struct metarray *
6275 metarray_find(struct rte_swx_pipeline *p, const char *name);
6276
6277 static int
6278 instr_metprefetch_translate(struct rte_swx_pipeline *p,
6279                             struct action *action,
6280                             char **tokens,
6281                             int n_tokens,
6282                             struct instruction *instr,
6283                             struct instruction_data *data __rte_unused)
6284 {
6285         char *metarray = tokens[1], *idx = tokens[2];
6286         struct metarray *m;
6287         struct field *fidx;
6288         uint32_t idx_struct_id, idx_val;
6289
6290         CHECK(n_tokens == 3, EINVAL);
6291
6292         m = metarray_find(p, metarray);
6293         CHECK(m, EINVAL);
6294
6295         /* METPREFETCH_H, METPREFETCH_M. */
6296         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6297         if (fidx) {
6298                 CHECK(!fidx->var_size, EINVAL);
6299
6300                 instr->type = INSTR_METPREFETCH_M;
6301                 if (idx[0] == 'h')
6302                         instr->type = INSTR_METPREFETCH_H;
6303
6304                 instr->meter.metarray_id = m->id;
6305                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6306                 instr->meter.idx.n_bits = fidx->n_bits;
6307                 instr->meter.idx.offset = fidx->offset / 8;
6308                 return 0;
6309         }
6310
6311         /* METPREFETCH_I. */
6312         idx_val = strtoul(idx, &idx, 0);
6313         CHECK(!idx[0], EINVAL);
6314
6315         instr->type = INSTR_METPREFETCH_I;
6316         instr->meter.metarray_id = m->id;
6317         instr->meter.idx_val = idx_val;
6318         return 0;
6319 }
6320
6321 static int
6322 instr_meter_translate(struct rte_swx_pipeline *p,
6323                       struct action *action,
6324                       char **tokens,
6325                       int n_tokens,
6326                       struct instruction *instr,
6327                       struct instruction_data *data __rte_unused)
6328 {
6329         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
6330         char *color_in = tokens[4], *color_out = tokens[5];
6331         struct metarray *m;
6332         struct field *fidx, *flength, *fcin, *fcout;
6333         uint32_t idx_struct_id, length_struct_id;
6334         uint32_t color_in_struct_id, color_out_struct_id;
6335
6336         CHECK(n_tokens == 6, EINVAL);
6337
6338         m = metarray_find(p, metarray);
6339         CHECK(m, EINVAL);
6340
6341         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6342
6343         flength = struct_field_parse(p, action, length, &length_struct_id);
6344         CHECK(flength, EINVAL);
6345         CHECK(!flength->var_size, EINVAL);
6346
6347         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
6348
6349         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
6350         CHECK(fcout, EINVAL);
6351         CHECK(!fcout->var_size, EINVAL);
6352
6353         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
6354         if (fidx && fcin) {
6355                 CHECK(!fidx->var_size, EINVAL);
6356                 CHECK(!fcin->var_size, EINVAL);
6357
6358                 instr->type = INSTR_METER_MMM;
6359                 if (idx[0] == 'h' && length[0] == 'h')
6360                         instr->type = INSTR_METER_HHM;
6361                 if (idx[0] == 'h' && length[0] != 'h')
6362                         instr->type = INSTR_METER_HMM;
6363                 if (idx[0] != 'h' && length[0] == 'h')
6364                         instr->type = INSTR_METER_MHM;
6365
6366                 instr->meter.metarray_id = m->id;
6367
6368                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6369                 instr->meter.idx.n_bits = fidx->n_bits;
6370                 instr->meter.idx.offset = fidx->offset / 8;
6371
6372                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6373                 instr->meter.length.n_bits = flength->n_bits;
6374                 instr->meter.length.offset = flength->offset / 8;
6375
6376                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6377                 instr->meter.color_in.n_bits = fcin->n_bits;
6378                 instr->meter.color_in.offset = fcin->offset / 8;
6379
6380                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6381                 instr->meter.color_out.n_bits = fcout->n_bits;
6382                 instr->meter.color_out.offset = fcout->offset / 8;
6383
6384                 return 0;
6385         }
6386
6387         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
6388         if (fidx && !fcin) {
6389                 uint32_t color_in_val;
6390
6391                 CHECK(!fidx->var_size, EINVAL);
6392
6393                 color_in_val = strtoul(color_in, &color_in, 0);
6394                 CHECK(!color_in[0], EINVAL);
6395
6396                 instr->type = INSTR_METER_MMI;
6397                 if (idx[0] == 'h' && length[0] == 'h')
6398                         instr->type = INSTR_METER_HHI;
6399                 if (idx[0] == 'h' && length[0] != 'h')
6400                         instr->type = INSTR_METER_HMI;
6401                 if (idx[0] != 'h' && length[0] == 'h')
6402                         instr->type = INSTR_METER_MHI;
6403
6404                 instr->meter.metarray_id = m->id;
6405
6406                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6407                 instr->meter.idx.n_bits = fidx->n_bits;
6408                 instr->meter.idx.offset = fidx->offset / 8;
6409
6410                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6411                 instr->meter.length.n_bits = flength->n_bits;
6412                 instr->meter.length.offset = flength->offset / 8;
6413
6414                 instr->meter.color_in_val = color_in_val;
6415
6416                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6417                 instr->meter.color_out.n_bits = fcout->n_bits;
6418                 instr->meter.color_out.offset = fcout->offset / 8;
6419
6420                 return 0;
6421         }
6422
6423         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
6424         if (!fidx && fcin) {
6425                 uint32_t idx_val;
6426
6427                 idx_val = strtoul(idx, &idx, 0);
6428                 CHECK(!idx[0], EINVAL);
6429
6430                 CHECK(!fcin->var_size, EINVAL);
6431
6432                 instr->type = INSTR_METER_IMM;
6433                 if (length[0] == 'h')
6434                         instr->type = INSTR_METER_IHM;
6435
6436                 instr->meter.metarray_id = m->id;
6437
6438                 instr->meter.idx_val = idx_val;
6439
6440                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6441                 instr->meter.length.n_bits = flength->n_bits;
6442                 instr->meter.length.offset = flength->offset / 8;
6443
6444                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6445                 instr->meter.color_in.n_bits = fcin->n_bits;
6446                 instr->meter.color_in.offset = fcin->offset / 8;
6447
6448                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6449                 instr->meter.color_out.n_bits = fcout->n_bits;
6450                 instr->meter.color_out.offset = fcout->offset / 8;
6451
6452                 return 0;
6453         }
6454
6455         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
6456         if (!fidx && !fcin) {
6457                 uint32_t idx_val, color_in_val;
6458
6459                 idx_val = strtoul(idx, &idx, 0);
6460                 CHECK(!idx[0], EINVAL);
6461
6462                 color_in_val = strtoul(color_in, &color_in, 0);
6463                 CHECK(!color_in[0], EINVAL);
6464
6465                 instr->type = INSTR_METER_IMI;
6466                 if (length[0] == 'h')
6467                         instr->type = INSTR_METER_IHI;
6468
6469                 instr->meter.metarray_id = m->id;
6470
6471                 instr->meter.idx_val = idx_val;
6472
6473                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6474                 instr->meter.length.n_bits = flength->n_bits;
6475                 instr->meter.length.offset = flength->offset / 8;
6476
6477                 instr->meter.color_in_val = color_in_val;
6478
6479                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6480                 instr->meter.color_out.n_bits = fcout->n_bits;
6481                 instr->meter.color_out.offset = fcout->offset / 8;
6482
6483                 return 0;
6484         }
6485
6486         CHECK(0, EINVAL);
6487 }
6488
6489 static inline struct meter *
6490 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6491 {
6492         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6493
6494         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6495         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6496         uint64_t idx64 = *idx64_ptr;
6497         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
6498         uint64_t idx = idx64 & idx64_mask & r->size_mask;
6499
6500         return &r->metarray[idx];
6501 }
6502
6503 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6504
6505 static inline struct meter *
6506 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6507 {
6508         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6509
6510         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6511         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6512         uint64_t idx64 = *idx64_ptr;
6513         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
6514
6515         return &r->metarray[idx];
6516 }
6517
6518 #else
6519
6520 #define instr_meter_idx_nbo instr_meter_idx_hbo
6521
6522 #endif
6523
6524 static inline struct meter *
6525 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
6526 {
6527         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6528
6529         uint64_t idx =  ip->meter.idx_val & r->size_mask;
6530
6531         return &r->metarray[idx];
6532 }
6533
6534 static inline uint32_t
6535 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
6536 {
6537         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6538         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6539         uint64_t src64 = *src64_ptr;
6540         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
6541         uint64_t src = src64 & src64_mask;
6542
6543         return (uint32_t)src;
6544 }
6545
6546 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6547
6548 static inline uint32_t
6549 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
6550 {
6551         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6552         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6553         uint64_t src64 = *src64_ptr;
6554         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
6555
6556         return (uint32_t)src;
6557 }
6558
6559 #else
6560
6561 #define instr_meter_length_nbo instr_meter_length_hbo
6562
6563 #endif
6564
6565 static inline enum rte_color
6566 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
6567 {
6568         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
6569         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
6570         uint64_t src64 = *src64_ptr;
6571         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
6572         uint64_t src = src64 & src64_mask;
6573
6574         return (enum rte_color)src;
6575 }
6576
6577 static inline void
6578 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
6579 {
6580         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
6581         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
6582         uint64_t dst64 = *dst64_ptr;
6583         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
6584
6585         uint64_t src = (uint64_t)color_out;
6586
6587         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6588 }
6589
6590 static inline void
6591 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
6592 {
6593         struct thread *t = &p->threads[p->thread_id];
6594         struct instruction *ip = t->ip;
6595         struct meter *m;
6596
6597         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
6598
6599         /* Structs. */
6600         m = instr_meter_idx_nbo(p, t, ip);
6601         rte_prefetch0(m);
6602
6603         /* Thread. */
6604         thread_ip_inc(p);
6605 }
6606
6607 static inline void
6608 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
6609 {
6610         struct thread *t = &p->threads[p->thread_id];
6611         struct instruction *ip = t->ip;
6612         struct meter *m;
6613
6614         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
6615
6616         /* Structs. */
6617         m = instr_meter_idx_hbo(p, t, ip);
6618         rte_prefetch0(m);
6619
6620         /* Thread. */
6621         thread_ip_inc(p);
6622 }
6623
6624 static inline void
6625 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
6626 {
6627         struct thread *t = &p->threads[p->thread_id];
6628         struct instruction *ip = t->ip;
6629         struct meter *m;
6630
6631         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
6632
6633         /* Structs. */
6634         m = instr_meter_idx_imm(p, ip);
6635         rte_prefetch0(m);
6636
6637         /* Thread. */
6638         thread_ip_inc(p);
6639 }
6640
6641 static inline void
6642 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
6643 {
6644         struct thread *t = &p->threads[p->thread_id];
6645         struct instruction *ip = t->ip;
6646         struct meter *m;
6647         uint64_t time, n_pkts, n_bytes;
6648         uint32_t length;
6649         enum rte_color color_in, color_out;
6650
6651         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
6652
6653         /* Structs. */
6654         m = instr_meter_idx_nbo(p, t, ip);
6655         rte_prefetch0(m->n_pkts);
6656         time = rte_get_tsc_cycles();
6657         length = instr_meter_length_nbo(t, ip);
6658         color_in = instr_meter_color_in_hbo(t, ip);
6659
6660         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6661                 &m->profile->profile,
6662                 time,
6663                 length,
6664                 color_in);
6665
6666         color_out &= m->color_mask;
6667
6668         n_pkts = m->n_pkts[color_out];
6669         n_bytes = m->n_bytes[color_out];
6670
6671         instr_meter_color_out_hbo_set(t, ip, color_out);
6672
6673         m->n_pkts[color_out] = n_pkts + 1;
6674         m->n_bytes[color_out] = n_bytes + length;
6675
6676         /* Thread. */
6677         thread_ip_inc(p);
6678 }
6679
6680 static inline void
6681 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
6682 {
6683         struct thread *t = &p->threads[p->thread_id];
6684         struct instruction *ip = t->ip;
6685         struct meter *m;
6686         uint64_t time, n_pkts, n_bytes;
6687         uint32_t length;
6688         enum rte_color color_in, color_out;
6689
6690         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
6691
6692         /* Structs. */
6693         m = instr_meter_idx_nbo(p, t, ip);
6694         rte_prefetch0(m->n_pkts);
6695         time = rte_get_tsc_cycles();
6696         length = instr_meter_length_nbo(t, ip);
6697         color_in = (enum rte_color)ip->meter.color_in_val;
6698
6699         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6700                 &m->profile->profile,
6701                 time,
6702                 length,
6703                 color_in);
6704
6705         color_out &= m->color_mask;
6706
6707         n_pkts = m->n_pkts[color_out];
6708         n_bytes = m->n_bytes[color_out];
6709
6710         instr_meter_color_out_hbo_set(t, ip, color_out);
6711
6712         m->n_pkts[color_out] = n_pkts + 1;
6713         m->n_bytes[color_out] = n_bytes + length;
6714
6715         /* Thread. */
6716         thread_ip_inc(p);
6717 }
6718
6719 static inline void
6720 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
6721 {
6722         struct thread *t = &p->threads[p->thread_id];
6723         struct instruction *ip = t->ip;
6724         struct meter *m;
6725         uint64_t time, n_pkts, n_bytes;
6726         uint32_t length;
6727         enum rte_color color_in, color_out;
6728
6729         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
6730
6731         /* Structs. */
6732         m = instr_meter_idx_nbo(p, t, ip);
6733         rte_prefetch0(m->n_pkts);
6734         time = rte_get_tsc_cycles();
6735         length = instr_meter_length_hbo(t, ip);
6736         color_in = instr_meter_color_in_hbo(t, ip);
6737
6738         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6739                 &m->profile->profile,
6740                 time,
6741                 length,
6742                 color_in);
6743
6744         color_out &= m->color_mask;
6745
6746         n_pkts = m->n_pkts[color_out];
6747         n_bytes = m->n_bytes[color_out];
6748
6749         instr_meter_color_out_hbo_set(t, ip, color_out);
6750
6751         m->n_pkts[color_out] = n_pkts + 1;
6752         m->n_bytes[color_out] = n_bytes + length;
6753
6754         /* Thread. */
6755         thread_ip_inc(p);
6756 }
6757 static inline void
6758 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
6759 {
6760         struct thread *t = &p->threads[p->thread_id];
6761         struct instruction *ip = t->ip;
6762         struct meter *m;
6763         uint64_t time, n_pkts, n_bytes;
6764         uint32_t length;
6765         enum rte_color color_in, color_out;
6766
6767         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
6768
6769         /* Structs. */
6770         m = instr_meter_idx_nbo(p, t, ip);
6771         rte_prefetch0(m->n_pkts);
6772         time = rte_get_tsc_cycles();
6773         length = instr_meter_length_hbo(t, ip);
6774         color_in = (enum rte_color)ip->meter.color_in_val;
6775
6776         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6777                 &m->profile->profile,
6778                 time,
6779                 length,
6780                 color_in);
6781
6782         color_out &= m->color_mask;
6783
6784         n_pkts = m->n_pkts[color_out];
6785         n_bytes = m->n_bytes[color_out];
6786
6787         instr_meter_color_out_hbo_set(t, ip, color_out);
6788
6789         m->n_pkts[color_out] = n_pkts + 1;
6790         m->n_bytes[color_out] = n_bytes + length;
6791
6792         /* Thread. */
6793         thread_ip_inc(p);
6794 }
6795
6796 static inline void
6797 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
6798 {
6799         struct thread *t = &p->threads[p->thread_id];
6800         struct instruction *ip = t->ip;
6801         struct meter *m;
6802         uint64_t time, n_pkts, n_bytes;
6803         uint32_t length;
6804         enum rte_color color_in, color_out;
6805
6806         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
6807
6808         /* Structs. */
6809         m = instr_meter_idx_hbo(p, t, ip);
6810         rte_prefetch0(m->n_pkts);
6811         time = rte_get_tsc_cycles();
6812         length = instr_meter_length_nbo(t, ip);
6813         color_in = instr_meter_color_in_hbo(t, ip);
6814
6815         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6816                 &m->profile->profile,
6817                 time,
6818                 length,
6819                 color_in);
6820
6821         color_out &= m->color_mask;
6822
6823         n_pkts = m->n_pkts[color_out];
6824         n_bytes = m->n_bytes[color_out];
6825
6826         instr_meter_color_out_hbo_set(t, ip, color_out);
6827
6828         m->n_pkts[color_out] = n_pkts + 1;
6829         m->n_bytes[color_out] = n_bytes + length;
6830
6831         /* Thread. */
6832         thread_ip_inc(p);
6833 }
6834
6835 static inline void
6836 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
6837 {
6838         struct thread *t = &p->threads[p->thread_id];
6839         struct instruction *ip = t->ip;
6840         struct meter *m;
6841         uint64_t time, n_pkts, n_bytes;
6842         uint32_t length;
6843         enum rte_color color_in, color_out;
6844
6845         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
6846
6847         /* Structs. */
6848         m = instr_meter_idx_hbo(p, t, ip);
6849         rte_prefetch0(m->n_pkts);
6850         time = rte_get_tsc_cycles();
6851         length = instr_meter_length_nbo(t, ip);
6852         color_in = (enum rte_color)ip->meter.color_in_val;
6853
6854         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6855                 &m->profile->profile,
6856                 time,
6857                 length,
6858                 color_in);
6859
6860         color_out &= m->color_mask;
6861
6862         n_pkts = m->n_pkts[color_out];
6863         n_bytes = m->n_bytes[color_out];
6864
6865         instr_meter_color_out_hbo_set(t, ip, color_out);
6866
6867         m->n_pkts[color_out] = n_pkts + 1;
6868         m->n_bytes[color_out] = n_bytes + length;
6869
6870         /* Thread. */
6871         thread_ip_inc(p);
6872 }
6873
6874 static inline void
6875 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
6876 {
6877         struct thread *t = &p->threads[p->thread_id];
6878         struct instruction *ip = t->ip;
6879         struct meter *m;
6880         uint64_t time, n_pkts, n_bytes;
6881         uint32_t length;
6882         enum rte_color color_in, color_out;
6883
6884         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
6885
6886         /* Structs. */
6887         m = instr_meter_idx_hbo(p, t, ip);
6888         rte_prefetch0(m->n_pkts);
6889         time = rte_get_tsc_cycles();
6890         length = instr_meter_length_hbo(t, ip);
6891         color_in = instr_meter_color_in_hbo(t, ip);
6892
6893         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6894                 &m->profile->profile,
6895                 time,
6896                 length,
6897                 color_in);
6898
6899         color_out &= m->color_mask;
6900
6901         n_pkts = m->n_pkts[color_out];
6902         n_bytes = m->n_bytes[color_out];
6903
6904         instr_meter_color_out_hbo_set(t, ip, color_out);
6905
6906         m->n_pkts[color_out] = n_pkts + 1;
6907         m->n_bytes[color_out] = n_bytes + length;
6908
6909         /* Thread. */
6910         thread_ip_inc(p);
6911 }
6912
6913 static inline void
6914 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
6915 {
6916         struct thread *t = &p->threads[p->thread_id];
6917         struct instruction *ip = t->ip;
6918         struct meter *m;
6919         uint64_t time, n_pkts, n_bytes;
6920         uint32_t length;
6921         enum rte_color color_in, color_out;
6922
6923         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
6924
6925         /* Structs. */
6926         m = instr_meter_idx_hbo(p, t, ip);
6927         rte_prefetch0(m->n_pkts);
6928         time = rte_get_tsc_cycles();
6929         length = instr_meter_length_hbo(t, ip);
6930         color_in = (enum rte_color)ip->meter.color_in_val;
6931
6932         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6933                 &m->profile->profile,
6934                 time,
6935                 length,
6936                 color_in);
6937
6938         color_out &= m->color_mask;
6939
6940         n_pkts = m->n_pkts[color_out];
6941         n_bytes = m->n_bytes[color_out];
6942
6943         instr_meter_color_out_hbo_set(t, ip, color_out);
6944
6945         m->n_pkts[color_out] = n_pkts + 1;
6946         m->n_bytes[color_out] = n_bytes + length;
6947
6948         /* Thread. */
6949         thread_ip_inc(p);
6950 }
6951
6952 static inline void
6953 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
6954 {
6955         struct thread *t = &p->threads[p->thread_id];
6956         struct instruction *ip = t->ip;
6957         struct meter *m;
6958         uint64_t time, n_pkts, n_bytes;
6959         uint32_t length;
6960         enum rte_color color_in, color_out;
6961
6962         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
6963
6964         /* Structs. */
6965         m = instr_meter_idx_imm(p, ip);
6966         rte_prefetch0(m->n_pkts);
6967         time = rte_get_tsc_cycles();
6968         length = instr_meter_length_nbo(t, ip);
6969         color_in = instr_meter_color_in_hbo(t, ip);
6970
6971         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6972                 &m->profile->profile,
6973                 time,
6974                 length,
6975                 color_in);
6976
6977         color_out &= m->color_mask;
6978
6979         n_pkts = m->n_pkts[color_out];
6980         n_bytes = m->n_bytes[color_out];
6981
6982         instr_meter_color_out_hbo_set(t, ip, color_out);
6983
6984         m->n_pkts[color_out] = n_pkts + 1;
6985         m->n_bytes[color_out] = n_bytes + length;
6986
6987         /* Thread. */
6988         thread_ip_inc(p);
6989 }
6990
6991 static inline void
6992 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
6993 {
6994         struct thread *t = &p->threads[p->thread_id];
6995         struct instruction *ip = t->ip;
6996         struct meter *m;
6997         uint64_t time, n_pkts, n_bytes;
6998         uint32_t length;
6999         enum rte_color color_in, color_out;
7000
7001         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
7002
7003         /* Structs. */
7004         m = instr_meter_idx_imm(p, ip);
7005         rte_prefetch0(m->n_pkts);
7006         time = rte_get_tsc_cycles();
7007         length = instr_meter_length_nbo(t, ip);
7008         color_in = (enum rte_color)ip->meter.color_in_val;
7009
7010         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7011                 &m->profile->profile,
7012                 time,
7013                 length,
7014                 color_in);
7015
7016         color_out &= m->color_mask;
7017
7018         n_pkts = m->n_pkts[color_out];
7019         n_bytes = m->n_bytes[color_out];
7020
7021         instr_meter_color_out_hbo_set(t, ip, color_out);
7022
7023         m->n_pkts[color_out] = n_pkts + 1;
7024         m->n_bytes[color_out] = n_bytes + length;
7025
7026         /* Thread. */
7027         thread_ip_inc(p);
7028 }
7029
7030 static inline void
7031 instr_meter_imm_exec(struct rte_swx_pipeline *p)
7032 {
7033         struct thread *t = &p->threads[p->thread_id];
7034         struct instruction *ip = t->ip;
7035         struct meter *m;
7036         uint64_t time, n_pkts, n_bytes;
7037         uint32_t length;
7038         enum rte_color color_in, color_out;
7039
7040         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
7041
7042         /* Structs. */
7043         m = instr_meter_idx_imm(p, ip);
7044         rte_prefetch0(m->n_pkts);
7045         time = rte_get_tsc_cycles();
7046         length = instr_meter_length_hbo(t, ip);
7047         color_in = instr_meter_color_in_hbo(t, ip);
7048
7049         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7050                 &m->profile->profile,
7051                 time,
7052                 length,
7053                 color_in);
7054
7055         color_out &= m->color_mask;
7056
7057         n_pkts = m->n_pkts[color_out];
7058         n_bytes = m->n_bytes[color_out];
7059
7060         instr_meter_color_out_hbo_set(t, ip, color_out);
7061
7062         m->n_pkts[color_out] = n_pkts + 1;
7063         m->n_bytes[color_out] = n_bytes + length;
7064
7065         /* Thread. */
7066         thread_ip_inc(p);
7067 }
7068 static inline void
7069 instr_meter_imi_exec(struct rte_swx_pipeline *p)
7070 {
7071         struct thread *t = &p->threads[p->thread_id];
7072         struct instruction *ip = t->ip;
7073         struct meter *m;
7074         uint64_t time, n_pkts, n_bytes;
7075         uint32_t length;
7076         enum rte_color color_in, color_out;
7077
7078         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
7079
7080         /* Structs. */
7081         m = instr_meter_idx_imm(p, ip);
7082         rte_prefetch0(m->n_pkts);
7083         time = rte_get_tsc_cycles();
7084         length = instr_meter_length_hbo(t, ip);
7085         color_in = (enum rte_color)ip->meter.color_in_val;
7086
7087         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7088                 &m->profile->profile,
7089                 time,
7090                 length,
7091                 color_in);
7092
7093         color_out &= m->color_mask;
7094
7095         n_pkts = m->n_pkts[color_out];
7096         n_bytes = m->n_bytes[color_out];
7097
7098         instr_meter_color_out_hbo_set(t, ip, color_out);
7099
7100         m->n_pkts[color_out] = n_pkts + 1;
7101         m->n_bytes[color_out] = n_bytes + length;
7102
7103         /* Thread. */
7104         thread_ip_inc(p);
7105 }
7106
7107 /*
7108  * jmp.
7109  */
7110 static struct action *
7111 action_find(struct rte_swx_pipeline *p, const char *name);
7112
7113 static int
7114 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
7115                     struct action *action __rte_unused,
7116                     char **tokens,
7117                     int n_tokens,
7118                     struct instruction *instr,
7119                     struct instruction_data *data)
7120 {
7121         CHECK(n_tokens == 2, EINVAL);
7122
7123         strcpy(data->jmp_label, tokens[1]);
7124
7125         instr->type = INSTR_JMP;
7126         instr->jmp.ip = NULL; /* Resolved later. */
7127         return 0;
7128 }
7129
7130 static int
7131 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
7132                           struct action *action __rte_unused,
7133                           char **tokens,
7134                           int n_tokens,
7135                           struct instruction *instr,
7136                           struct instruction_data *data)
7137 {
7138         struct header *h;
7139
7140         CHECK(n_tokens == 3, EINVAL);
7141
7142         strcpy(data->jmp_label, tokens[1]);
7143
7144         h = header_parse(p, tokens[2]);
7145         CHECK(h, EINVAL);
7146
7147         instr->type = INSTR_JMP_VALID;
7148         instr->jmp.ip = NULL; /* Resolved later. */
7149         instr->jmp.header_id = h->id;
7150         return 0;
7151 }
7152
7153 static int
7154 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
7155                             struct action *action __rte_unused,
7156                             char **tokens,
7157                             int n_tokens,
7158                             struct instruction *instr,
7159                             struct instruction_data *data)
7160 {
7161         struct header *h;
7162
7163         CHECK(n_tokens == 3, EINVAL);
7164
7165         strcpy(data->jmp_label, tokens[1]);
7166
7167         h = header_parse(p, tokens[2]);
7168         CHECK(h, EINVAL);
7169
7170         instr->type = INSTR_JMP_INVALID;
7171         instr->jmp.ip = NULL; /* Resolved later. */
7172         instr->jmp.header_id = h->id;
7173         return 0;
7174 }
7175
7176 static int
7177 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
7178                         struct action *action,
7179                         char **tokens,
7180                         int n_tokens,
7181                         struct instruction *instr,
7182                         struct instruction_data *data)
7183 {
7184         CHECK(!action, EINVAL);
7185         CHECK(n_tokens == 2, EINVAL);
7186
7187         strcpy(data->jmp_label, tokens[1]);
7188
7189         instr->type = INSTR_JMP_HIT;
7190         instr->jmp.ip = NULL; /* Resolved later. */
7191         return 0;
7192 }
7193
7194 static int
7195 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
7196                          struct action *action,
7197                          char **tokens,
7198                          int n_tokens,
7199                          struct instruction *instr,
7200                          struct instruction_data *data)
7201 {
7202         CHECK(!action, EINVAL);
7203         CHECK(n_tokens == 2, EINVAL);
7204
7205         strcpy(data->jmp_label, tokens[1]);
7206
7207         instr->type = INSTR_JMP_MISS;
7208         instr->jmp.ip = NULL; /* Resolved later. */
7209         return 0;
7210 }
7211
7212 static int
7213 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
7214                                struct action *action,
7215                                char **tokens,
7216                                int n_tokens,
7217                                struct instruction *instr,
7218                                struct instruction_data *data)
7219 {
7220         struct action *a;
7221
7222         CHECK(!action, EINVAL);
7223         CHECK(n_tokens == 3, EINVAL);
7224
7225         strcpy(data->jmp_label, tokens[1]);
7226
7227         a = action_find(p, tokens[2]);
7228         CHECK(a, EINVAL);
7229
7230         instr->type = INSTR_JMP_ACTION_HIT;
7231         instr->jmp.ip = NULL; /* Resolved later. */
7232         instr->jmp.action_id = a->id;
7233         return 0;
7234 }
7235
7236 static int
7237 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
7238                                 struct action *action,
7239                                 char **tokens,
7240                                 int n_tokens,
7241                                 struct instruction *instr,
7242                                 struct instruction_data *data)
7243 {
7244         struct action *a;
7245
7246         CHECK(!action, EINVAL);
7247         CHECK(n_tokens == 3, EINVAL);
7248
7249         strcpy(data->jmp_label, tokens[1]);
7250
7251         a = action_find(p, tokens[2]);
7252         CHECK(a, EINVAL);
7253
7254         instr->type = INSTR_JMP_ACTION_MISS;
7255         instr->jmp.ip = NULL; /* Resolved later. */
7256         instr->jmp.action_id = a->id;
7257         return 0;
7258 }
7259
7260 static int
7261 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
7262                        struct action *action,
7263                        char **tokens,
7264                        int n_tokens,
7265                        struct instruction *instr,
7266                        struct instruction_data *data)
7267 {
7268         char *a = tokens[2], *b = tokens[3];
7269         struct field *fa, *fb;
7270         uint64_t b_val;
7271         uint32_t a_struct_id, b_struct_id;
7272
7273         CHECK(n_tokens == 4, EINVAL);
7274
7275         strcpy(data->jmp_label, tokens[1]);
7276
7277         fa = struct_field_parse(p, action, a, &a_struct_id);
7278         CHECK(fa, EINVAL);
7279         CHECK(!fa->var_size, EINVAL);
7280
7281         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
7282         fb = struct_field_parse(p, action, b, &b_struct_id);
7283         if (fb) {
7284                 CHECK(!fb->var_size, EINVAL);
7285
7286                 instr->type = INSTR_JMP_EQ;
7287                 if (a[0] != 'h' && b[0] == 'h')
7288                         instr->type = INSTR_JMP_EQ_MH;
7289                 if (a[0] == 'h' && b[0] != 'h')
7290                         instr->type = INSTR_JMP_EQ_HM;
7291                 if (a[0] == 'h' && b[0] == 'h')
7292                         instr->type = INSTR_JMP_EQ_HH;
7293                 instr->jmp.ip = NULL; /* Resolved later. */
7294
7295                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7296                 instr->jmp.a.n_bits = fa->n_bits;
7297                 instr->jmp.a.offset = fa->offset / 8;
7298                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7299                 instr->jmp.b.n_bits = fb->n_bits;
7300                 instr->jmp.b.offset = fb->offset / 8;
7301                 return 0;
7302         }
7303
7304         /* JMP_EQ_I. */
7305         b_val = strtoull(b, &b, 0);
7306         CHECK(!b[0], EINVAL);
7307
7308         if (a[0] == 'h')
7309                 b_val = hton64(b_val) >> (64 - fa->n_bits);
7310
7311         instr->type = INSTR_JMP_EQ_I;
7312         instr->jmp.ip = NULL; /* Resolved later. */
7313         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7314         instr->jmp.a.n_bits = fa->n_bits;
7315         instr->jmp.a.offset = fa->offset / 8;
7316         instr->jmp.b_val = b_val;
7317         return 0;
7318 }
7319
7320 static int
7321 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
7322                         struct action *action,
7323                         char **tokens,
7324                         int n_tokens,
7325                         struct instruction *instr,
7326                         struct instruction_data *data)
7327 {
7328         char *a = tokens[2], *b = tokens[3];
7329         struct field *fa, *fb;
7330         uint64_t b_val;
7331         uint32_t a_struct_id, b_struct_id;
7332
7333         CHECK(n_tokens == 4, EINVAL);
7334
7335         strcpy(data->jmp_label, tokens[1]);
7336
7337         fa = struct_field_parse(p, action, a, &a_struct_id);
7338         CHECK(fa, EINVAL);
7339         CHECK(!fa->var_size, EINVAL);
7340
7341         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
7342         fb = struct_field_parse(p, action, b, &b_struct_id);
7343         if (fb) {
7344                 CHECK(!fb->var_size, EINVAL);
7345
7346                 instr->type = INSTR_JMP_NEQ;
7347                 if (a[0] != 'h' && b[0] == 'h')
7348                         instr->type = INSTR_JMP_NEQ_MH;
7349                 if (a[0] == 'h' && b[0] != 'h')
7350                         instr->type = INSTR_JMP_NEQ_HM;
7351                 if (a[0] == 'h' && b[0] == 'h')
7352                         instr->type = INSTR_JMP_NEQ_HH;
7353                 instr->jmp.ip = NULL; /* Resolved later. */
7354
7355                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7356                 instr->jmp.a.n_bits = fa->n_bits;
7357                 instr->jmp.a.offset = fa->offset / 8;
7358                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7359                 instr->jmp.b.n_bits = fb->n_bits;
7360                 instr->jmp.b.offset = fb->offset / 8;
7361                 return 0;
7362         }
7363
7364         /* JMP_NEQ_I. */
7365         b_val = strtoull(b, &b, 0);
7366         CHECK(!b[0], EINVAL);
7367
7368         if (a[0] == 'h')
7369                 b_val = hton64(b_val) >> (64 - fa->n_bits);
7370
7371         instr->type = INSTR_JMP_NEQ_I;
7372         instr->jmp.ip = NULL; /* Resolved later. */
7373         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7374         instr->jmp.a.n_bits = fa->n_bits;
7375         instr->jmp.a.offset = fa->offset / 8;
7376         instr->jmp.b_val = b_val;
7377         return 0;
7378 }
7379
7380 static int
7381 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
7382                        struct action *action,
7383                        char **tokens,
7384                        int n_tokens,
7385                        struct instruction *instr,
7386                        struct instruction_data *data)
7387 {
7388         char *a = tokens[2], *b = tokens[3];
7389         struct field *fa, *fb;
7390         uint64_t b_val;
7391         uint32_t a_struct_id, b_struct_id;
7392
7393         CHECK(n_tokens == 4, EINVAL);
7394
7395         strcpy(data->jmp_label, tokens[1]);
7396
7397         fa = struct_field_parse(p, action, a, &a_struct_id);
7398         CHECK(fa, EINVAL);
7399         CHECK(!fa->var_size, EINVAL);
7400
7401         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
7402         fb = struct_field_parse(p, action, b, &b_struct_id);
7403         if (fb) {
7404                 CHECK(!fb->var_size, EINVAL);
7405
7406                 instr->type = INSTR_JMP_LT;
7407                 if (a[0] == 'h' && b[0] != 'h')
7408                         instr->type = INSTR_JMP_LT_HM;
7409                 if (a[0] != 'h' && b[0] == 'h')
7410                         instr->type = INSTR_JMP_LT_MH;
7411                 if (a[0] == 'h' && b[0] == 'h')
7412                         instr->type = INSTR_JMP_LT_HH;
7413                 instr->jmp.ip = NULL; /* Resolved later. */
7414
7415                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7416                 instr->jmp.a.n_bits = fa->n_bits;
7417                 instr->jmp.a.offset = fa->offset / 8;
7418                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7419                 instr->jmp.b.n_bits = fb->n_bits;
7420                 instr->jmp.b.offset = fb->offset / 8;
7421                 return 0;
7422         }
7423
7424         /* JMP_LT_MI, JMP_LT_HI. */
7425         b_val = strtoull(b, &b, 0);
7426         CHECK(!b[0], EINVAL);
7427
7428         instr->type = INSTR_JMP_LT_MI;
7429         if (a[0] == 'h')
7430                 instr->type = INSTR_JMP_LT_HI;
7431         instr->jmp.ip = NULL; /* Resolved later. */
7432
7433         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7434         instr->jmp.a.n_bits = fa->n_bits;
7435         instr->jmp.a.offset = fa->offset / 8;
7436         instr->jmp.b_val = b_val;
7437         return 0;
7438 }
7439
7440 static int
7441 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
7442                        struct action *action,
7443                        char **tokens,
7444                        int n_tokens,
7445                        struct instruction *instr,
7446                        struct instruction_data *data)
7447 {
7448         char *a = tokens[2], *b = tokens[3];
7449         struct field *fa, *fb;
7450         uint64_t b_val;
7451         uint32_t a_struct_id, b_struct_id;
7452
7453         CHECK(n_tokens == 4, EINVAL);
7454
7455         strcpy(data->jmp_label, tokens[1]);
7456
7457         fa = struct_field_parse(p, action, a, &a_struct_id);
7458         CHECK(fa, EINVAL);
7459         CHECK(!fa->var_size, EINVAL);
7460
7461         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
7462         fb = struct_field_parse(p, action, b, &b_struct_id);
7463         if (fb) {
7464                 CHECK(!fb->var_size, EINVAL);
7465
7466                 instr->type = INSTR_JMP_GT;
7467                 if (a[0] == 'h' && b[0] != 'h')
7468                         instr->type = INSTR_JMP_GT_HM;
7469                 if (a[0] != 'h' && b[0] == 'h')
7470                         instr->type = INSTR_JMP_GT_MH;
7471                 if (a[0] == 'h' && b[0] == 'h')
7472                         instr->type = INSTR_JMP_GT_HH;
7473                 instr->jmp.ip = NULL; /* Resolved later. */
7474
7475                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7476                 instr->jmp.a.n_bits = fa->n_bits;
7477                 instr->jmp.a.offset = fa->offset / 8;
7478                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7479                 instr->jmp.b.n_bits = fb->n_bits;
7480                 instr->jmp.b.offset = fb->offset / 8;
7481                 return 0;
7482         }
7483
7484         /* JMP_GT_MI, JMP_GT_HI. */
7485         b_val = strtoull(b, &b, 0);
7486         CHECK(!b[0], EINVAL);
7487
7488         instr->type = INSTR_JMP_GT_MI;
7489         if (a[0] == 'h')
7490                 instr->type = INSTR_JMP_GT_HI;
7491         instr->jmp.ip = NULL; /* Resolved later. */
7492
7493         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7494         instr->jmp.a.n_bits = fa->n_bits;
7495         instr->jmp.a.offset = fa->offset / 8;
7496         instr->jmp.b_val = b_val;
7497         return 0;
7498 }
7499
7500 static inline void
7501 instr_jmp_exec(struct rte_swx_pipeline *p)
7502 {
7503         struct thread *t = &p->threads[p->thread_id];
7504         struct instruction *ip = t->ip;
7505
7506         TRACE("[Thread %2u] jmp\n", p->thread_id);
7507
7508         thread_ip_set(t, ip->jmp.ip);
7509 }
7510
7511 static inline void
7512 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
7513 {
7514         struct thread *t = &p->threads[p->thread_id];
7515         struct instruction *ip = t->ip;
7516         uint32_t header_id = ip->jmp.header_id;
7517
7518         TRACE("[Thread %2u] jmpv\n", p->thread_id);
7519
7520         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
7521 }
7522
7523 static inline void
7524 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
7525 {
7526         struct thread *t = &p->threads[p->thread_id];
7527         struct instruction *ip = t->ip;
7528         uint32_t header_id = ip->jmp.header_id;
7529
7530         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
7531
7532         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
7533 }
7534
7535 static inline void
7536 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
7537 {
7538         struct thread *t = &p->threads[p->thread_id];
7539         struct instruction *ip = t->ip;
7540         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
7541
7542         TRACE("[Thread %2u] jmph\n", p->thread_id);
7543
7544         t->ip = ip_next[t->hit];
7545 }
7546
7547 static inline void
7548 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
7549 {
7550         struct thread *t = &p->threads[p->thread_id];
7551         struct instruction *ip = t->ip;
7552         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
7553
7554         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
7555
7556         t->ip = ip_next[t->hit];
7557 }
7558
7559 static inline void
7560 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
7561 {
7562         struct thread *t = &p->threads[p->thread_id];
7563         struct instruction *ip = t->ip;
7564
7565         TRACE("[Thread %2u] jmpa\n", p->thread_id);
7566
7567         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
7568 }
7569
7570 static inline void
7571 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
7572 {
7573         struct thread *t = &p->threads[p->thread_id];
7574         struct instruction *ip = t->ip;
7575
7576         TRACE("[Thread %2u] jmpna\n", p->thread_id);
7577
7578         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
7579 }
7580
7581 static inline void
7582 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
7583 {
7584         struct thread *t = &p->threads[p->thread_id];
7585         struct instruction *ip = t->ip;
7586
7587         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
7588
7589         JMP_CMP(t, ip, ==);
7590 }
7591
7592 static inline void
7593 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
7594 {
7595         struct thread *t = &p->threads[p->thread_id];
7596         struct instruction *ip = t->ip;
7597
7598         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
7599
7600         JMP_CMP_MH(t, ip, ==);
7601 }
7602
7603 static inline void
7604 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
7605 {
7606         struct thread *t = &p->threads[p->thread_id];
7607         struct instruction *ip = t->ip;
7608
7609         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
7610
7611         JMP_CMP_HM(t, ip, ==);
7612 }
7613
7614 static inline void
7615 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
7616 {
7617         struct thread *t = &p->threads[p->thread_id];
7618         struct instruction *ip = t->ip;
7619
7620         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
7621
7622         JMP_CMP_HH_FAST(t, ip, ==);
7623 }
7624
7625 static inline void
7626 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
7627 {
7628         struct thread *t = &p->threads[p->thread_id];
7629         struct instruction *ip = t->ip;
7630
7631         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
7632
7633         JMP_CMP_I(t, ip, ==);
7634 }
7635
7636 static inline void
7637 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
7638 {
7639         struct thread *t = &p->threads[p->thread_id];
7640         struct instruction *ip = t->ip;
7641
7642         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
7643
7644         JMP_CMP(t, ip, !=);
7645 }
7646
7647 static inline void
7648 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
7649 {
7650         struct thread *t = &p->threads[p->thread_id];
7651         struct instruction *ip = t->ip;
7652
7653         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
7654
7655         JMP_CMP_MH(t, ip, !=);
7656 }
7657
7658 static inline void
7659 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
7660 {
7661         struct thread *t = &p->threads[p->thread_id];
7662         struct instruction *ip = t->ip;
7663
7664         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
7665
7666         JMP_CMP_HM(t, ip, !=);
7667 }
7668
7669 static inline void
7670 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
7671 {
7672         struct thread *t = &p->threads[p->thread_id];
7673         struct instruction *ip = t->ip;
7674
7675         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
7676
7677         JMP_CMP_HH_FAST(t, ip, !=);
7678 }
7679
7680 static inline void
7681 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
7682 {
7683         struct thread *t = &p->threads[p->thread_id];
7684         struct instruction *ip = t->ip;
7685
7686         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
7687
7688         JMP_CMP_I(t, ip, !=);
7689 }
7690
7691 static inline void
7692 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
7693 {
7694         struct thread *t = &p->threads[p->thread_id];
7695         struct instruction *ip = t->ip;
7696
7697         TRACE("[Thread %2u] jmplt\n", p->thread_id);
7698
7699         JMP_CMP(t, ip, <);
7700 }
7701
7702 static inline void
7703 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
7704 {
7705         struct thread *t = &p->threads[p->thread_id];
7706         struct instruction *ip = t->ip;
7707
7708         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
7709
7710         JMP_CMP_MH(t, ip, <);
7711 }
7712
7713 static inline void
7714 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
7715 {
7716         struct thread *t = &p->threads[p->thread_id];
7717         struct instruction *ip = t->ip;
7718
7719         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
7720
7721         JMP_CMP_HM(t, ip, <);
7722 }
7723
7724 static inline void
7725 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
7726 {
7727         struct thread *t = &p->threads[p->thread_id];
7728         struct instruction *ip = t->ip;
7729
7730         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
7731
7732         JMP_CMP_HH(t, ip, <);
7733 }
7734
7735 static inline void
7736 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
7737 {
7738         struct thread *t = &p->threads[p->thread_id];
7739         struct instruction *ip = t->ip;
7740
7741         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
7742
7743         JMP_CMP_MI(t, ip, <);
7744 }
7745
7746 static inline void
7747 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
7748 {
7749         struct thread *t = &p->threads[p->thread_id];
7750         struct instruction *ip = t->ip;
7751
7752         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
7753
7754         JMP_CMP_HI(t, ip, <);
7755 }
7756
7757 static inline void
7758 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
7759 {
7760         struct thread *t = &p->threads[p->thread_id];
7761         struct instruction *ip = t->ip;
7762
7763         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
7764
7765         JMP_CMP(t, ip, >);
7766 }
7767
7768 static inline void
7769 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
7770 {
7771         struct thread *t = &p->threads[p->thread_id];
7772         struct instruction *ip = t->ip;
7773
7774         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
7775
7776         JMP_CMP_MH(t, ip, >);
7777 }
7778
7779 static inline void
7780 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
7781 {
7782         struct thread *t = &p->threads[p->thread_id];
7783         struct instruction *ip = t->ip;
7784
7785         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
7786
7787         JMP_CMP_HM(t, ip, >);
7788 }
7789
7790 static inline void
7791 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
7792 {
7793         struct thread *t = &p->threads[p->thread_id];
7794         struct instruction *ip = t->ip;
7795
7796         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
7797
7798         JMP_CMP_HH(t, ip, >);
7799 }
7800
7801 static inline void
7802 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
7803 {
7804         struct thread *t = &p->threads[p->thread_id];
7805         struct instruction *ip = t->ip;
7806
7807         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
7808
7809         JMP_CMP_MI(t, ip, >);
7810 }
7811
7812 static inline void
7813 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
7814 {
7815         struct thread *t = &p->threads[p->thread_id];
7816         struct instruction *ip = t->ip;
7817
7818         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
7819
7820         JMP_CMP_HI(t, ip, >);
7821 }
7822
7823 /*
7824  * return.
7825  */
7826 static int
7827 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
7828                        struct action *action,
7829                        char **tokens __rte_unused,
7830                        int n_tokens,
7831                        struct instruction *instr,
7832                        struct instruction_data *data __rte_unused)
7833 {
7834         CHECK(action, EINVAL);
7835         CHECK(n_tokens == 1, EINVAL);
7836
7837         instr->type = INSTR_RETURN;
7838         return 0;
7839 }
7840
7841 static inline void
7842 instr_return_exec(struct rte_swx_pipeline *p)
7843 {
7844         struct thread *t = &p->threads[p->thread_id];
7845
7846         TRACE("[Thread %2u] return\n", p->thread_id);
7847
7848         t->ip = t->ret;
7849 }
7850
7851 static int
7852 instr_translate(struct rte_swx_pipeline *p,
7853                 struct action *action,
7854                 char *string,
7855                 struct instruction *instr,
7856                 struct instruction_data *data)
7857 {
7858         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
7859         int n_tokens = 0, tpos = 0;
7860
7861         /* Parse the instruction string into tokens. */
7862         for ( ; ; ) {
7863                 char *token;
7864
7865                 token = strtok_r(string, " \t\v", &string);
7866                 if (!token)
7867                         break;
7868
7869                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
7870                 CHECK_NAME(token, EINVAL);
7871
7872                 tokens[n_tokens] = token;
7873                 n_tokens++;
7874         }
7875
7876         CHECK(n_tokens, EINVAL);
7877
7878         /* Handle the optional instruction label. */
7879         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
7880                 strcpy(data->label, tokens[0]);
7881
7882                 tpos += 2;
7883                 CHECK(n_tokens - tpos, EINVAL);
7884         }
7885
7886         /* Identify the instruction type. */
7887         if (!strcmp(tokens[tpos], "rx"))
7888                 return instr_rx_translate(p,
7889                                           action,
7890                                           &tokens[tpos],
7891                                           n_tokens - tpos,
7892                                           instr,
7893                                           data);
7894
7895         if (!strcmp(tokens[tpos], "tx"))
7896                 return instr_tx_translate(p,
7897                                           action,
7898                                           &tokens[tpos],
7899                                           n_tokens - tpos,
7900                                           instr,
7901                                           data);
7902
7903         if (!strcmp(tokens[tpos], "drop"))
7904                 return instr_drop_translate(p,
7905                                             action,
7906                                             &tokens[tpos],
7907                                             n_tokens - tpos,
7908                                             instr,
7909                                             data);
7910
7911         if (!strcmp(tokens[tpos], "extract"))
7912                 return instr_hdr_extract_translate(p,
7913                                                    action,
7914                                                    &tokens[tpos],
7915                                                    n_tokens - tpos,
7916                                                    instr,
7917                                                    data);
7918
7919         if (!strcmp(tokens[tpos], "emit"))
7920                 return instr_hdr_emit_translate(p,
7921                                                 action,
7922                                                 &tokens[tpos],
7923                                                 n_tokens - tpos,
7924                                                 instr,
7925                                                 data);
7926
7927         if (!strcmp(tokens[tpos], "validate"))
7928                 return instr_hdr_validate_translate(p,
7929                                                     action,
7930                                                     &tokens[tpos],
7931                                                     n_tokens - tpos,
7932                                                     instr,
7933                                                     data);
7934
7935         if (!strcmp(tokens[tpos], "invalidate"))
7936                 return instr_hdr_invalidate_translate(p,
7937                                                       action,
7938                                                       &tokens[tpos],
7939                                                       n_tokens - tpos,
7940                                                       instr,
7941                                                       data);
7942
7943         if (!strcmp(tokens[tpos], "mov"))
7944                 return instr_mov_translate(p,
7945                                            action,
7946                                            &tokens[tpos],
7947                                            n_tokens - tpos,
7948                                            instr,
7949                                            data);
7950
7951         if (!strcmp(tokens[tpos], "add"))
7952                 return instr_alu_add_translate(p,
7953                                                action,
7954                                                &tokens[tpos],
7955                                                n_tokens - tpos,
7956                                                instr,
7957                                                data);
7958
7959         if (!strcmp(tokens[tpos], "sub"))
7960                 return instr_alu_sub_translate(p,
7961                                                action,
7962                                                &tokens[tpos],
7963                                                n_tokens - tpos,
7964                                                instr,
7965                                                data);
7966
7967         if (!strcmp(tokens[tpos], "ckadd"))
7968                 return instr_alu_ckadd_translate(p,
7969                                                  action,
7970                                                  &tokens[tpos],
7971                                                  n_tokens - tpos,
7972                                                  instr,
7973                                                  data);
7974
7975         if (!strcmp(tokens[tpos], "cksub"))
7976                 return instr_alu_cksub_translate(p,
7977                                                  action,
7978                                                  &tokens[tpos],
7979                                                  n_tokens - tpos,
7980                                                  instr,
7981                                                  data);
7982
7983         if (!strcmp(tokens[tpos], "and"))
7984                 return instr_alu_and_translate(p,
7985                                                action,
7986                                                &tokens[tpos],
7987                                                n_tokens - tpos,
7988                                                instr,
7989                                                data);
7990
7991         if (!strcmp(tokens[tpos], "or"))
7992                 return instr_alu_or_translate(p,
7993                                               action,
7994                                               &tokens[tpos],
7995                                               n_tokens - tpos,
7996                                               instr,
7997                                               data);
7998
7999         if (!strcmp(tokens[tpos], "xor"))
8000                 return instr_alu_xor_translate(p,
8001                                                action,
8002                                                &tokens[tpos],
8003                                                n_tokens - tpos,
8004                                                instr,
8005                                                data);
8006
8007         if (!strcmp(tokens[tpos], "shl"))
8008                 return instr_alu_shl_translate(p,
8009                                                action,
8010                                                &tokens[tpos],
8011                                                n_tokens - tpos,
8012                                                instr,
8013                                                data);
8014
8015         if (!strcmp(tokens[tpos], "shr"))
8016                 return instr_alu_shr_translate(p,
8017                                                action,
8018                                                &tokens[tpos],
8019                                                n_tokens - tpos,
8020                                                instr,
8021                                                data);
8022
8023         if (!strcmp(tokens[tpos], "regprefetch"))
8024                 return instr_regprefetch_translate(p,
8025                                                    action,
8026                                                    &tokens[tpos],
8027                                                    n_tokens - tpos,
8028                                                    instr,
8029                                                    data);
8030
8031         if (!strcmp(tokens[tpos], "regrd"))
8032                 return instr_regrd_translate(p,
8033                                              action,
8034                                              &tokens[tpos],
8035                                              n_tokens - tpos,
8036                                              instr,
8037                                              data);
8038
8039         if (!strcmp(tokens[tpos], "regwr"))
8040                 return instr_regwr_translate(p,
8041                                              action,
8042                                              &tokens[tpos],
8043                                              n_tokens - tpos,
8044                                              instr,
8045                                              data);
8046
8047         if (!strcmp(tokens[tpos], "regadd"))
8048                 return instr_regadd_translate(p,
8049                                               action,
8050                                               &tokens[tpos],
8051                                               n_tokens - tpos,
8052                                               instr,
8053                                               data);
8054
8055         if (!strcmp(tokens[tpos], "metprefetch"))
8056                 return instr_metprefetch_translate(p,
8057                                                    action,
8058                                                    &tokens[tpos],
8059                                                    n_tokens - tpos,
8060                                                    instr,
8061                                                    data);
8062
8063         if (!strcmp(tokens[tpos], "meter"))
8064                 return instr_meter_translate(p,
8065                                              action,
8066                                              &tokens[tpos],
8067                                              n_tokens - tpos,
8068                                              instr,
8069                                              data);
8070
8071         if (!strcmp(tokens[tpos], "table"))
8072                 return instr_table_translate(p,
8073                                              action,
8074                                              &tokens[tpos],
8075                                              n_tokens - tpos,
8076                                              instr,
8077                                              data);
8078
8079         if (!strcmp(tokens[tpos], "extern"))
8080                 return instr_extern_translate(p,
8081                                               action,
8082                                               &tokens[tpos],
8083                                               n_tokens - tpos,
8084                                               instr,
8085                                               data);
8086
8087         if (!strcmp(tokens[tpos], "jmp"))
8088                 return instr_jmp_translate(p,
8089                                            action,
8090                                            &tokens[tpos],
8091                                            n_tokens - tpos,
8092                                            instr,
8093                                            data);
8094
8095         if (!strcmp(tokens[tpos], "jmpv"))
8096                 return instr_jmp_valid_translate(p,
8097                                                  action,
8098                                                  &tokens[tpos],
8099                                                  n_tokens - tpos,
8100                                                  instr,
8101                                                  data);
8102
8103         if (!strcmp(tokens[tpos], "jmpnv"))
8104                 return instr_jmp_invalid_translate(p,
8105                                                    action,
8106                                                    &tokens[tpos],
8107                                                    n_tokens - tpos,
8108                                                    instr,
8109                                                    data);
8110
8111         if (!strcmp(tokens[tpos], "jmph"))
8112                 return instr_jmp_hit_translate(p,
8113                                                action,
8114                                                &tokens[tpos],
8115                                                n_tokens - tpos,
8116                                                instr,
8117                                                data);
8118
8119         if (!strcmp(tokens[tpos], "jmpnh"))
8120                 return instr_jmp_miss_translate(p,
8121                                                 action,
8122                                                 &tokens[tpos],
8123                                                 n_tokens - tpos,
8124                                                 instr,
8125                                                 data);
8126
8127         if (!strcmp(tokens[tpos], "jmpa"))
8128                 return instr_jmp_action_hit_translate(p,
8129                                                       action,
8130                                                       &tokens[tpos],
8131                                                       n_tokens - tpos,
8132                                                       instr,
8133                                                       data);
8134
8135         if (!strcmp(tokens[tpos], "jmpna"))
8136                 return instr_jmp_action_miss_translate(p,
8137                                                        action,
8138                                                        &tokens[tpos],
8139                                                        n_tokens - tpos,
8140                                                        instr,
8141                                                        data);
8142
8143         if (!strcmp(tokens[tpos], "jmpeq"))
8144                 return instr_jmp_eq_translate(p,
8145                                               action,
8146                                               &tokens[tpos],
8147                                               n_tokens - tpos,
8148                                               instr,
8149                                               data);
8150
8151         if (!strcmp(tokens[tpos], "jmpneq"))
8152                 return instr_jmp_neq_translate(p,
8153                                                action,
8154                                                &tokens[tpos],
8155                                                n_tokens - tpos,
8156                                                instr,
8157                                                data);
8158
8159         if (!strcmp(tokens[tpos], "jmplt"))
8160                 return instr_jmp_lt_translate(p,
8161                                               action,
8162                                               &tokens[tpos],
8163                                               n_tokens - tpos,
8164                                               instr,
8165                                               data);
8166
8167         if (!strcmp(tokens[tpos], "jmpgt"))
8168                 return instr_jmp_gt_translate(p,
8169                                               action,
8170                                               &tokens[tpos],
8171                                               n_tokens - tpos,
8172                                               instr,
8173                                               data);
8174
8175         if (!strcmp(tokens[tpos], "return"))
8176                 return instr_return_translate(p,
8177                                               action,
8178                                               &tokens[tpos],
8179                                               n_tokens - tpos,
8180                                               instr,
8181                                               data);
8182
8183         CHECK(0, EINVAL);
8184 }
8185
8186 static struct instruction_data *
8187 label_find(struct instruction_data *data, uint32_t n, const char *label)
8188 {
8189         uint32_t i;
8190
8191         for (i = 0; i < n; i++)
8192                 if (!strcmp(label, data[i].label))
8193                         return &data[i];
8194
8195         return NULL;
8196 }
8197
8198 static uint32_t
8199 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
8200 {
8201         uint32_t count = 0, i;
8202
8203         if (!label[0])
8204                 return 0;
8205
8206         for (i = 0; i < n; i++)
8207                 if (!strcmp(label, data[i].jmp_label))
8208                         count++;
8209
8210         return count;
8211 }
8212
8213 static int
8214 instr_label_check(struct instruction_data *instruction_data,
8215                   uint32_t n_instructions)
8216 {
8217         uint32_t i;
8218
8219         /* Check that all instruction labels are unique. */
8220         for (i = 0; i < n_instructions; i++) {
8221                 struct instruction_data *data = &instruction_data[i];
8222                 char *label = data->label;
8223                 uint32_t j;
8224
8225                 if (!label[0])
8226                         continue;
8227
8228                 for (j = i + 1; j < n_instructions; j++)
8229                         CHECK(strcmp(label, data[j].label), EINVAL);
8230         }
8231
8232         /* Get users for each instruction label. */
8233         for (i = 0; i < n_instructions; i++) {
8234                 struct instruction_data *data = &instruction_data[i];
8235                 char *label = data->label;
8236
8237                 data->n_users = label_is_used(instruction_data,
8238                                               n_instructions,
8239                                               label);
8240         }
8241
8242         return 0;
8243 }
8244
8245 static int
8246 instr_jmp_resolve(struct instruction *instructions,
8247                   struct instruction_data *instruction_data,
8248                   uint32_t n_instructions)
8249 {
8250         uint32_t i;
8251
8252         for (i = 0; i < n_instructions; i++) {
8253                 struct instruction *instr = &instructions[i];
8254                 struct instruction_data *data = &instruction_data[i];
8255                 struct instruction_data *found;
8256
8257                 if (!instruction_is_jmp(instr))
8258                         continue;
8259
8260                 found = label_find(instruction_data,
8261                                    n_instructions,
8262                                    data->jmp_label);
8263                 CHECK(found, EINVAL);
8264
8265                 instr->jmp.ip = &instructions[found - instruction_data];
8266         }
8267
8268         return 0;
8269 }
8270
8271 static int
8272 instr_verify(struct rte_swx_pipeline *p __rte_unused,
8273              struct action *a,
8274              struct instruction *instr,
8275              struct instruction_data *data __rte_unused,
8276              uint32_t n_instructions)
8277 {
8278         if (!a) {
8279                 enum instruction_type type;
8280                 uint32_t i;
8281
8282                 /* Check that the first instruction is rx. */
8283                 CHECK(instr[0].type == INSTR_RX, EINVAL);
8284
8285                 /* Check that there is at least one tx instruction. */
8286                 for (i = 0; i < n_instructions; i++) {
8287                         type = instr[i].type;
8288
8289                         if (instruction_is_tx(type))
8290                                 break;
8291                 }
8292                 CHECK(i < n_instructions, EINVAL);
8293
8294                 /* Check that the last instruction is either tx or unconditional
8295                  * jump.
8296                  */
8297                 type = instr[n_instructions - 1].type;
8298                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
8299         }
8300
8301         if (a) {
8302                 enum instruction_type type;
8303                 uint32_t i;
8304
8305                 /* Check that there is at least one return or tx instruction. */
8306                 for (i = 0; i < n_instructions; i++) {
8307                         type = instr[i].type;
8308
8309                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
8310                                 break;
8311                 }
8312                 CHECK(i < n_instructions, EINVAL);
8313         }
8314
8315         return 0;
8316 }
8317
8318 static uint32_t
8319 instr_compact(struct instruction *instructions,
8320               struct instruction_data *instruction_data,
8321               uint32_t n_instructions)
8322 {
8323         uint32_t i, pos = 0;
8324
8325         /* Eliminate the invalid instructions that have been optimized out. */
8326         for (i = 0; i < n_instructions; i++) {
8327                 struct instruction *instr = &instructions[i];
8328                 struct instruction_data *data = &instruction_data[i];
8329
8330                 if (data->invalid)
8331                         continue;
8332
8333                 if (i != pos) {
8334                         memcpy(&instructions[pos], instr, sizeof(*instr));
8335                         memcpy(&instruction_data[pos], data, sizeof(*data));
8336                 }
8337
8338                 pos++;
8339         }
8340
8341         return pos;
8342 }
8343
8344 static int
8345 instr_pattern_extract_many_search(struct instruction *instr,
8346                                   struct instruction_data *data,
8347                                   uint32_t n_instr,
8348                                   uint32_t *n_pattern_instr)
8349 {
8350         uint32_t i;
8351
8352         for (i = 0; i < n_instr; i++) {
8353                 if (data[i].invalid)
8354                         break;
8355
8356                 if (instr[i].type != INSTR_HDR_EXTRACT)
8357                         break;
8358
8359                 if (i == RTE_DIM(instr->io.hdr.header_id))
8360                         break;
8361
8362                 if (i && data[i].n_users)
8363                         break;
8364         }
8365
8366         if (i < 2)
8367                 return 0;
8368
8369         *n_pattern_instr = i;
8370         return 1;
8371 }
8372
8373 static void
8374 instr_pattern_extract_many_replace(struct instruction *instr,
8375                                    struct instruction_data *data,
8376                                    uint32_t n_instr)
8377 {
8378         uint32_t i;
8379
8380         for (i = 1; i < n_instr; i++) {
8381                 instr[0].type++;
8382                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8383                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8384                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8385
8386                 data[i].invalid = 1;
8387         }
8388 }
8389
8390 static uint32_t
8391 instr_pattern_extract_many_optimize(struct instruction *instructions,
8392                                     struct instruction_data *instruction_data,
8393                                     uint32_t n_instructions)
8394 {
8395         uint32_t i;
8396
8397         for (i = 0; i < n_instructions; ) {
8398                 struct instruction *instr = &instructions[i];
8399                 struct instruction_data *data = &instruction_data[i];
8400                 uint32_t n_instr = 0;
8401                 int detected;
8402
8403                 /* Extract many. */
8404                 detected = instr_pattern_extract_many_search(instr,
8405                                                              data,
8406                                                              n_instructions - i,
8407                                                              &n_instr);
8408                 if (detected) {
8409                         instr_pattern_extract_many_replace(instr,
8410                                                            data,
8411                                                            n_instr);
8412                         i += n_instr;
8413                         continue;
8414                 }
8415
8416                 /* No pattern starting at the current instruction. */
8417                 i++;
8418         }
8419
8420         /* Eliminate the invalid instructions that have been optimized out. */
8421         n_instructions = instr_compact(instructions,
8422                                        instruction_data,
8423                                        n_instructions);
8424
8425         return n_instructions;
8426 }
8427
8428 static int
8429 instr_pattern_emit_many_tx_search(struct instruction *instr,
8430                                   struct instruction_data *data,
8431                                   uint32_t n_instr,
8432                                   uint32_t *n_pattern_instr)
8433 {
8434         uint32_t i;
8435
8436         for (i = 0; i < n_instr; i++) {
8437                 if (data[i].invalid)
8438                         break;
8439
8440                 if (instr[i].type != INSTR_HDR_EMIT)
8441                         break;
8442
8443                 if (i == RTE_DIM(instr->io.hdr.header_id))
8444                         break;
8445
8446                 if (i && data[i].n_users)
8447                         break;
8448         }
8449
8450         if (!i)
8451                 return 0;
8452
8453         if (!instruction_is_tx(instr[i].type))
8454                 return 0;
8455
8456         if (data[i].n_users)
8457                 return 0;
8458
8459         i++;
8460
8461         *n_pattern_instr = i;
8462         return 1;
8463 }
8464
8465 static void
8466 instr_pattern_emit_many_tx_replace(struct instruction *instr,
8467                                    struct instruction_data *data,
8468                                    uint32_t n_instr)
8469 {
8470         uint32_t i;
8471
8472         /* Any emit instruction in addition to the first one. */
8473         for (i = 1; i < n_instr - 1; i++) {
8474                 instr[0].type++;
8475                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8476                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8477                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8478
8479                 data[i].invalid = 1;
8480         }
8481
8482         /* The TX instruction is the last one in the pattern. */
8483         instr[0].type++;
8484         instr[0].io.io.offset = instr[i].io.io.offset;
8485         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
8486         data[i].invalid = 1;
8487 }
8488
8489 static uint32_t
8490 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
8491                                     struct instruction_data *instruction_data,
8492                                     uint32_t n_instructions)
8493 {
8494         uint32_t i;
8495
8496         for (i = 0; i < n_instructions; ) {
8497                 struct instruction *instr = &instructions[i];
8498                 struct instruction_data *data = &instruction_data[i];
8499                 uint32_t n_instr = 0;
8500                 int detected;
8501
8502                 /* Emit many + TX. */
8503                 detected = instr_pattern_emit_many_tx_search(instr,
8504                                                              data,
8505                                                              n_instructions - i,
8506                                                              &n_instr);
8507                 if (detected) {
8508                         instr_pattern_emit_many_tx_replace(instr,
8509                                                            data,
8510                                                            n_instr);
8511                         i += n_instr;
8512                         continue;
8513                 }
8514
8515                 /* No pattern starting at the current instruction. */
8516                 i++;
8517         }
8518
8519         /* Eliminate the invalid instructions that have been optimized out. */
8520         n_instructions = instr_compact(instructions,
8521                                        instruction_data,
8522                                        n_instructions);
8523
8524         return n_instructions;
8525 }
8526
8527 static uint32_t
8528 action_arg_src_mov_count(struct action *a,
8529                          uint32_t arg_id,
8530                          struct instruction *instructions,
8531                          struct instruction_data *instruction_data,
8532                          uint32_t n_instructions);
8533
8534 static int
8535 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
8536                                       struct action *a,
8537                                       struct instruction *instr,
8538                                       struct instruction_data *data,
8539                                       uint32_t n_instr,
8540                                       struct instruction *instructions,
8541                                       struct instruction_data *instruction_data,
8542                                       uint32_t n_instructions,
8543                                       uint32_t *n_pattern_instr)
8544 {
8545         struct header *h;
8546         uint32_t src_field_id, i, j;
8547
8548         /* Prerequisites. */
8549         if (!a || !a->st)
8550                 return 0;
8551
8552         /* First instruction: MOV_HM. */
8553         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
8554                 return 0;
8555
8556         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
8557         if (!h || h->st->var_size)
8558                 return 0;
8559
8560         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
8561                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
8562                         break;
8563
8564         if (src_field_id == a->st->n_fields)
8565                 return 0;
8566
8567         if (instr[0].mov.dst.offset ||
8568             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
8569             instr[0].mov.src.struct_id ||
8570             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
8571             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
8572                 return 0;
8573
8574         if ((n_instr < h->st->n_fields + 1) ||
8575              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
8576                 return 0;
8577
8578         /* Subsequent instructions: MOV_HM. */
8579         for (i = 1; i < h->st->n_fields; i++)
8580                 if (data[i].invalid ||
8581                     data[i].n_users ||
8582                     (instr[i].type != INSTR_MOV_HM) ||
8583                     (instr[i].mov.dst.struct_id != h->struct_id) ||
8584                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
8585                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
8586                     instr[i].mov.src.struct_id ||
8587                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
8588                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
8589                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
8590                         return 0;
8591
8592         /* Last instruction: HDR_VALIDATE. */
8593         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
8594             (instr[i].valid.header_id != h->id))
8595                 return 0;
8596
8597         /* Check that none of the action args that are used as source for this
8598          * DMA transfer are not used as source in any other mov instruction.
8599          */
8600         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
8601                 uint32_t n_users;
8602
8603                 n_users = action_arg_src_mov_count(a,
8604                                                    j,
8605                                                    instructions,
8606                                                    instruction_data,
8607                                                    n_instructions);
8608                 if (n_users > 1)
8609                         return 0;
8610         }
8611
8612         *n_pattern_instr = 1 + i;
8613         return 1;
8614 }
8615
8616 static void
8617 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
8618                                        struct action *a,
8619                                        struct instruction *instr,
8620                                        struct instruction_data *data,
8621                                        uint32_t n_instr)
8622 {
8623         struct header *h;
8624         uint32_t src_field_id, src_offset, i;
8625
8626         /* Read from the instructions before they are modified. */
8627         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
8628         if (!h)
8629                 return;
8630
8631         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
8632                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
8633                         break;
8634
8635         if (src_field_id == a->st->n_fields)
8636                 return;
8637
8638         src_offset = instr[0].mov.src.offset;
8639
8640         /* Modify the instructions. */
8641         instr[0].type = INSTR_DMA_HT;
8642         instr[0].dma.dst.header_id[0] = h->id;
8643         instr[0].dma.dst.struct_id[0] = h->struct_id;
8644         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
8645         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
8646
8647         for (i = 1; i < n_instr; i++)
8648                 data[i].invalid = 1;
8649
8650         /* Update the endianness of the action arguments to header endianness. */
8651         for (i = 0; i < h->st->n_fields; i++)
8652                 a->args_endianness[src_field_id + i] = 1;
8653 }
8654
8655 static uint32_t
8656 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
8657                                         struct action *a,
8658                                         struct instruction *instructions,
8659                                         struct instruction_data *instruction_data,
8660                                         uint32_t n_instructions)
8661 {
8662         uint32_t i;
8663
8664         if (!a || !a->st)
8665                 return n_instructions;
8666
8667         for (i = 0; i < n_instructions; ) {
8668                 struct instruction *instr = &instructions[i];
8669                 struct instruction_data *data = &instruction_data[i];
8670                 uint32_t n_instr = 0;
8671                 int detected;
8672
8673                 /* Mov all + validate. */
8674                 detected = instr_pattern_mov_all_validate_search(p,
8675                                                                  a,
8676                                                                  instr,
8677                                                                  data,
8678                                                                  n_instructions - i,
8679                                                                  instructions,
8680                                                                  instruction_data,
8681                                                                  n_instructions,
8682                                                                  &n_instr);
8683                 if (detected) {
8684                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
8685                         i += n_instr;
8686                         continue;
8687                 }
8688
8689                 /* No pattern starting at the current instruction. */
8690                 i++;
8691         }
8692
8693         /* Eliminate the invalid instructions that have been optimized out. */
8694         n_instructions = instr_compact(instructions,
8695                                        instruction_data,
8696                                        n_instructions);
8697
8698         return n_instructions;
8699 }
8700
8701 static int
8702 instr_pattern_dma_many_search(struct instruction *instr,
8703                               struct instruction_data *data,
8704                               uint32_t n_instr,
8705                               uint32_t *n_pattern_instr)
8706 {
8707         uint32_t i;
8708
8709         for (i = 0; i < n_instr; i++) {
8710                 if (data[i].invalid)
8711                         break;
8712
8713                 if (instr[i].type != INSTR_DMA_HT)
8714                         break;
8715
8716                 if (i == RTE_DIM(instr->dma.dst.header_id))
8717                         break;
8718
8719                 if (i && data[i].n_users)
8720                         break;
8721         }
8722
8723         if (i < 2)
8724                 return 0;
8725
8726         *n_pattern_instr = i;
8727         return 1;
8728 }
8729
8730 static void
8731 instr_pattern_dma_many_replace(struct instruction *instr,
8732                                struct instruction_data *data,
8733                                uint32_t n_instr)
8734 {
8735         uint32_t i;
8736
8737         for (i = 1; i < n_instr; i++) {
8738                 instr[0].type++;
8739                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
8740                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
8741                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
8742                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
8743
8744                 data[i].invalid = 1;
8745         }
8746 }
8747
8748 static uint32_t
8749 instr_pattern_dma_many_optimize(struct instruction *instructions,
8750                struct instruction_data *instruction_data,
8751                uint32_t n_instructions)
8752 {
8753         uint32_t i;
8754
8755         for (i = 0; i < n_instructions; ) {
8756                 struct instruction *instr = &instructions[i];
8757                 struct instruction_data *data = &instruction_data[i];
8758                 uint32_t n_instr = 0;
8759                 int detected;
8760
8761                 /* DMA many. */
8762                 detected = instr_pattern_dma_many_search(instr,
8763                                                          data,
8764                                                          n_instructions - i,
8765                                                          &n_instr);
8766                 if (detected) {
8767                         instr_pattern_dma_many_replace(instr, data, n_instr);
8768                         i += n_instr;
8769                         continue;
8770                 }
8771
8772                 /* No pattern starting at the current instruction. */
8773                 i++;
8774         }
8775
8776         /* Eliminate the invalid instructions that have been optimized out. */
8777         n_instructions = instr_compact(instructions,
8778                                        instruction_data,
8779                                        n_instructions);
8780
8781         return n_instructions;
8782 }
8783
8784 static uint32_t
8785 instr_optimize(struct rte_swx_pipeline *p,
8786                struct action *a,
8787                struct instruction *instructions,
8788                struct instruction_data *instruction_data,
8789                uint32_t n_instructions)
8790 {
8791         /* Extract many. */
8792         n_instructions = instr_pattern_extract_many_optimize(instructions,
8793                                                              instruction_data,
8794                                                              n_instructions);
8795
8796         /* Emit many + TX. */
8797         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
8798                                                              instruction_data,
8799                                                              n_instructions);
8800
8801         /* Mov all + validate. */
8802         n_instructions = instr_pattern_mov_all_validate_optimize(p,
8803                                                                  a,
8804                                                                  instructions,
8805                                                                  instruction_data,
8806                                                                  n_instructions);
8807
8808         /* DMA many. */
8809         n_instructions = instr_pattern_dma_many_optimize(instructions,
8810                                                          instruction_data,
8811                                                          n_instructions);
8812
8813         return n_instructions;
8814 }
8815
8816 static int
8817 instruction_config(struct rte_swx_pipeline *p,
8818                    struct action *a,
8819                    const char **instructions,
8820                    uint32_t n_instructions)
8821 {
8822         struct instruction *instr = NULL;
8823         struct instruction_data *data = NULL;
8824         int err = 0;
8825         uint32_t i;
8826
8827         CHECK(n_instructions, EINVAL);
8828         CHECK(instructions, EINVAL);
8829         for (i = 0; i < n_instructions; i++)
8830                 CHECK_INSTRUCTION(instructions[i], EINVAL);
8831
8832         /* Memory allocation. */
8833         instr = calloc(n_instructions, sizeof(struct instruction));
8834         if (!instr) {
8835                 err = -ENOMEM;
8836                 goto error;
8837         }
8838
8839         data = calloc(n_instructions, sizeof(struct instruction_data));
8840         if (!data) {
8841                 err = -ENOMEM;
8842                 goto error;
8843         }
8844
8845         for (i = 0; i < n_instructions; i++) {
8846                 char *string = strdup(instructions[i]);
8847                 if (!string) {
8848                         err = -ENOMEM;
8849                         goto error;
8850                 }
8851
8852                 err = instr_translate(p, a, string, &instr[i], &data[i]);
8853                 if (err) {
8854                         free(string);
8855                         goto error;
8856                 }
8857
8858                 free(string);
8859         }
8860
8861         err = instr_label_check(data, n_instructions);
8862         if (err)
8863                 goto error;
8864
8865         err = instr_verify(p, a, instr, data, n_instructions);
8866         if (err)
8867                 goto error;
8868
8869         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
8870
8871         err = instr_jmp_resolve(instr, data, n_instructions);
8872         if (err)
8873                 goto error;
8874
8875         if (a) {
8876                 a->instructions = instr;
8877                 a->n_instructions = n_instructions;
8878         } else {
8879                 p->instructions = instr;
8880                 p->n_instructions = n_instructions;
8881         }
8882
8883         free(data);
8884         return 0;
8885
8886 error:
8887         free(data);
8888         free(instr);
8889         return err;
8890 }
8891
8892 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
8893
8894 static instr_exec_t instruction_table[] = {
8895         [INSTR_RX] = instr_rx_exec,
8896         [INSTR_TX] = instr_tx_exec,
8897         [INSTR_TX_I] = instr_tx_i_exec,
8898
8899         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
8900         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
8901         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
8902         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
8903         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
8904         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
8905         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
8906         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
8907         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
8908
8909         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
8910         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
8911         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
8912         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
8913         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
8914         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
8915         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
8916         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
8917         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
8918
8919         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
8920         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
8921
8922         [INSTR_MOV] = instr_mov_exec,
8923         [INSTR_MOV_MH] = instr_mov_mh_exec,
8924         [INSTR_MOV_HM] = instr_mov_hm_exec,
8925         [INSTR_MOV_HH] = instr_mov_hh_exec,
8926         [INSTR_MOV_I] = instr_mov_i_exec,
8927
8928         [INSTR_DMA_HT] = instr_dma_ht_exec,
8929         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
8930         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
8931         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
8932         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
8933         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
8934         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
8935         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
8936
8937         [INSTR_ALU_ADD] = instr_alu_add_exec,
8938         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
8939         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
8940         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
8941         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
8942         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
8943
8944         [INSTR_ALU_SUB] = instr_alu_sub_exec,
8945         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
8946         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
8947         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
8948         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
8949         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
8950
8951         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
8952         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
8953         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
8954         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
8955
8956         [INSTR_ALU_AND] = instr_alu_and_exec,
8957         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
8958         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
8959         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
8960         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
8961
8962         [INSTR_ALU_OR] = instr_alu_or_exec,
8963         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
8964         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
8965         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
8966         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
8967
8968         [INSTR_ALU_XOR] = instr_alu_xor_exec,
8969         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
8970         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
8971         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
8972         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
8973
8974         [INSTR_ALU_SHL] = instr_alu_shl_exec,
8975         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
8976         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
8977         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
8978         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
8979         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
8980
8981         [INSTR_ALU_SHR] = instr_alu_shr_exec,
8982         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
8983         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
8984         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
8985         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
8986         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
8987
8988         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
8989         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
8990         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
8991
8992         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
8993         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
8994         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
8995         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
8996         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
8997         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
8998
8999         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
9000         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
9001         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
9002         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
9003         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
9004         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
9005         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
9006         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
9007         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
9008
9009         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
9010         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
9011         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
9012         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
9013         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
9014         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
9015         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
9016         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
9017         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
9018
9019         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
9020         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
9021         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
9022
9023         [INSTR_METER_HHM] = instr_meter_hhm_exec,
9024         [INSTR_METER_HHI] = instr_meter_hhi_exec,
9025         [INSTR_METER_HMM] = instr_meter_hmm_exec,
9026         [INSTR_METER_HMI] = instr_meter_hmi_exec,
9027         [INSTR_METER_MHM] = instr_meter_mhm_exec,
9028         [INSTR_METER_MHI] = instr_meter_mhi_exec,
9029         [INSTR_METER_MMM] = instr_meter_mmm_exec,
9030         [INSTR_METER_MMI] = instr_meter_mmi_exec,
9031         [INSTR_METER_IHM] = instr_meter_ihm_exec,
9032         [INSTR_METER_IHI] = instr_meter_ihi_exec,
9033         [INSTR_METER_IMM] = instr_meter_imm_exec,
9034         [INSTR_METER_IMI] = instr_meter_imi_exec,
9035
9036         [INSTR_TABLE] = instr_table_exec,
9037         [INSTR_SELECTOR] = instr_selector_exec,
9038         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
9039         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
9040
9041         [INSTR_JMP] = instr_jmp_exec,
9042         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
9043         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
9044         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
9045         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
9046         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
9047         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
9048
9049         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
9050         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
9051         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
9052         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
9053         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
9054
9055         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
9056         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
9057         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
9058         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
9059         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
9060
9061         [INSTR_JMP_LT] = instr_jmp_lt_exec,
9062         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
9063         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
9064         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
9065         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
9066         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
9067
9068         [INSTR_JMP_GT] = instr_jmp_gt_exec,
9069         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
9070         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
9071         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
9072         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
9073         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
9074
9075         [INSTR_RETURN] = instr_return_exec,
9076 };
9077
9078 static inline void
9079 instr_exec(struct rte_swx_pipeline *p)
9080 {
9081         struct thread *t = &p->threads[p->thread_id];
9082         struct instruction *ip = t->ip;
9083         instr_exec_t instr = instruction_table[ip->type];
9084
9085         instr(p);
9086 }
9087
9088 /*
9089  * Action.
9090  */
9091 static struct action *
9092 action_find(struct rte_swx_pipeline *p, const char *name)
9093 {
9094         struct action *elem;
9095
9096         if (!name)
9097                 return NULL;
9098
9099         TAILQ_FOREACH(elem, &p->actions, node)
9100                 if (strcmp(elem->name, name) == 0)
9101                         return elem;
9102
9103         return NULL;
9104 }
9105
9106 static struct action *
9107 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9108 {
9109         struct action *action = NULL;
9110
9111         TAILQ_FOREACH(action, &p->actions, node)
9112                 if (action->id == id)
9113                         return action;
9114
9115         return NULL;
9116 }
9117
9118 static struct field *
9119 action_field_find(struct action *a, const char *name)
9120 {
9121         return a->st ? struct_type_field_find(a->st, name) : NULL;
9122 }
9123
9124 static struct field *
9125 action_field_parse(struct action *action, const char *name)
9126 {
9127         if (name[0] != 't' || name[1] != '.')
9128                 return NULL;
9129
9130         return action_field_find(action, &name[2]);
9131 }
9132
9133 int
9134 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
9135                                const char *name,
9136                                const char *args_struct_type_name,
9137                                const char **instructions,
9138                                uint32_t n_instructions)
9139 {
9140         struct struct_type *args_struct_type = NULL;
9141         struct action *a;
9142         int err;
9143
9144         CHECK(p, EINVAL);
9145
9146         CHECK_NAME(name, EINVAL);
9147         CHECK(!action_find(p, name), EEXIST);
9148
9149         if (args_struct_type_name) {
9150                 CHECK_NAME(args_struct_type_name, EINVAL);
9151                 args_struct_type = struct_type_find(p, args_struct_type_name);
9152                 CHECK(args_struct_type, EINVAL);
9153                 CHECK(!args_struct_type->var_size, EINVAL);
9154         }
9155
9156         /* Node allocation. */
9157         a = calloc(1, sizeof(struct action));
9158         CHECK(a, ENOMEM);
9159         if (args_struct_type) {
9160                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
9161                 if (!a->args_endianness) {
9162                         free(a);
9163                         CHECK(0, ENOMEM);
9164                 }
9165         }
9166
9167         /* Node initialization. */
9168         strcpy(a->name, name);
9169         a->st = args_struct_type;
9170         a->id = p->n_actions;
9171
9172         /* Instruction translation. */
9173         err = instruction_config(p, a, instructions, n_instructions);
9174         if (err) {
9175                 free(a->args_endianness);
9176                 free(a);
9177                 return err;
9178         }
9179
9180         /* Node add to tailq. */
9181         TAILQ_INSERT_TAIL(&p->actions, a, node);
9182         p->n_actions++;
9183
9184         return 0;
9185 }
9186
9187 static int
9188 action_build(struct rte_swx_pipeline *p)
9189 {
9190         struct action *action;
9191
9192         p->action_instructions = calloc(p->n_actions,
9193                                         sizeof(struct instruction *));
9194         CHECK(p->action_instructions, ENOMEM);
9195
9196         TAILQ_FOREACH(action, &p->actions, node)
9197                 p->action_instructions[action->id] = action->instructions;
9198
9199         return 0;
9200 }
9201
9202 static void
9203 action_build_free(struct rte_swx_pipeline *p)
9204 {
9205         free(p->action_instructions);
9206         p->action_instructions = NULL;
9207 }
9208
9209 static void
9210 action_free(struct rte_swx_pipeline *p)
9211 {
9212         action_build_free(p);
9213
9214         for ( ; ; ) {
9215                 struct action *action;
9216
9217                 action = TAILQ_FIRST(&p->actions);
9218                 if (!action)
9219                         break;
9220
9221                 TAILQ_REMOVE(&p->actions, action, node);
9222                 free(action->instructions);
9223                 free(action);
9224         }
9225 }
9226
9227 static uint32_t
9228 action_arg_src_mov_count(struct action *a,
9229                          uint32_t arg_id,
9230                          struct instruction *instructions,
9231                          struct instruction_data *instruction_data,
9232                          uint32_t n_instructions)
9233 {
9234         uint32_t offset, n_users = 0, i;
9235
9236         if (!a->st ||
9237             (arg_id >= a->st->n_fields) ||
9238             !instructions ||
9239             !instruction_data ||
9240             !n_instructions)
9241                 return 0;
9242
9243         offset = a->st->fields[arg_id].offset / 8;
9244
9245         for (i = 0; i < n_instructions; i++) {
9246                 struct instruction *instr = &instructions[i];
9247                 struct instruction_data *data = &instruction_data[i];
9248
9249                 if (data->invalid ||
9250                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
9251                     instr->mov.src.struct_id ||
9252                     (instr->mov.src.offset != offset))
9253                         continue;
9254
9255                 n_users++;
9256         }
9257
9258         return n_users;
9259 }
9260
9261 /*
9262  * Table.
9263  */
9264 static struct table_type *
9265 table_type_find(struct rte_swx_pipeline *p, const char *name)
9266 {
9267         struct table_type *elem;
9268
9269         TAILQ_FOREACH(elem, &p->table_types, node)
9270                 if (strcmp(elem->name, name) == 0)
9271                         return elem;
9272
9273         return NULL;
9274 }
9275
9276 static struct table_type *
9277 table_type_resolve(struct rte_swx_pipeline *p,
9278                    const char *recommended_type_name,
9279                    enum rte_swx_table_match_type match_type)
9280 {
9281         struct table_type *elem;
9282
9283         /* Only consider the recommended type if the match type is correct. */
9284         if (recommended_type_name)
9285                 TAILQ_FOREACH(elem, &p->table_types, node)
9286                         if (!strcmp(elem->name, recommended_type_name) &&
9287                             (elem->match_type == match_type))
9288                                 return elem;
9289
9290         /* Ignore the recommended type and get the first element with this match
9291          * type.
9292          */
9293         TAILQ_FOREACH(elem, &p->table_types, node)
9294                 if (elem->match_type == match_type)
9295                         return elem;
9296
9297         return NULL;
9298 }
9299
9300 static struct table *
9301 table_find(struct rte_swx_pipeline *p, const char *name)
9302 {
9303         struct table *elem;
9304
9305         TAILQ_FOREACH(elem, &p->tables, node)
9306                 if (strcmp(elem->name, name) == 0)
9307                         return elem;
9308
9309         return NULL;
9310 }
9311
9312 static struct table *
9313 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9314 {
9315         struct table *table = NULL;
9316
9317         TAILQ_FOREACH(table, &p->tables, node)
9318                 if (table->id == id)
9319                         return table;
9320
9321         return NULL;
9322 }
9323
9324 int
9325 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
9326                                      const char *name,
9327                                      enum rte_swx_table_match_type match_type,
9328                                      struct rte_swx_table_ops *ops)
9329 {
9330         struct table_type *elem;
9331
9332         CHECK(p, EINVAL);
9333
9334         CHECK_NAME(name, EINVAL);
9335         CHECK(!table_type_find(p, name), EEXIST);
9336
9337         CHECK(ops, EINVAL);
9338         CHECK(ops->create, EINVAL);
9339         CHECK(ops->lkp, EINVAL);
9340         CHECK(ops->free, EINVAL);
9341
9342         /* Node allocation. */
9343         elem = calloc(1, sizeof(struct table_type));
9344         CHECK(elem, ENOMEM);
9345
9346         /* Node initialization. */
9347         strcpy(elem->name, name);
9348         elem->match_type = match_type;
9349         memcpy(&elem->ops, ops, sizeof(*ops));
9350
9351         /* Node add to tailq. */
9352         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
9353
9354         return 0;
9355 }
9356
9357 static int
9358 table_match_type_resolve(struct rte_swx_match_field_params *fields,
9359                          uint32_t n_fields,
9360                          enum rte_swx_table_match_type *match_type)
9361 {
9362         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
9363
9364         for (i = 0; i < n_fields; i++) {
9365                 struct rte_swx_match_field_params  *f = &fields[i];
9366
9367                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
9368                         n_fields_em++;
9369
9370                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
9371                         n_fields_lpm++;
9372         }
9373
9374         if ((n_fields_lpm > 1) ||
9375             (n_fields_lpm && (n_fields_em != n_fields - 1)))
9376                 return -EINVAL;
9377
9378         *match_type = (n_fields_em == n_fields) ?
9379                        RTE_SWX_TABLE_MATCH_EXACT :
9380                        RTE_SWX_TABLE_MATCH_WILDCARD;
9381
9382         return 0;
9383 }
9384
9385 static int
9386 table_match_fields_check(struct rte_swx_pipeline *p,
9387                          struct rte_swx_pipeline_table_params *params,
9388                          struct header **header)
9389 {
9390         struct header *h0 = NULL;
9391         struct field *hf, *mf;
9392         uint32_t *offset = NULL, i;
9393         int status = 0;
9394
9395         /* Return if no match fields. */
9396         if (!params->n_fields) {
9397                 if (params->fields) {
9398                         status = -EINVAL;
9399                         goto end;
9400                 }
9401
9402                 if (header)
9403                         *header = NULL;
9404
9405                 return 0;
9406         }
9407
9408         /* Memory allocation. */
9409         offset = calloc(params->n_fields, sizeof(uint32_t));
9410         if (!offset) {
9411                 status = -ENOMEM;
9412                 goto end;
9413         }
9414
9415         /* Check that all the match fields belong to either the same header or
9416          * to the meta-data.
9417          */
9418         hf = header_field_parse(p, params->fields[0].name, &h0);
9419         mf = metadata_field_parse(p, params->fields[0].name);
9420         if ((!hf && !mf) || (hf && hf->var_size)) {
9421                 status = -EINVAL;
9422                 goto end;
9423         }
9424
9425         offset[0] = h0 ? hf->offset : mf->offset;
9426
9427         for (i = 1; i < params->n_fields; i++)
9428                 if (h0) {
9429                         struct header *h;
9430
9431                         hf = header_field_parse(p, params->fields[i].name, &h);
9432                         if (!hf || (h->id != h0->id) || hf->var_size) {
9433                                 status = -EINVAL;
9434                                 goto end;
9435                         }
9436
9437                         offset[i] = hf->offset;
9438                 } else {
9439                         mf = metadata_field_parse(p, params->fields[i].name);
9440                         if (!mf) {
9441                                 status = -EINVAL;
9442                                 goto end;
9443                         }
9444
9445                         offset[i] = mf->offset;
9446                 }
9447
9448         /* Check that there are no duplicated match fields. */
9449         for (i = 0; i < params->n_fields; i++) {
9450                 uint32_t j;
9451
9452                 for (j = 0; j < i; j++)
9453                         if (offset[j] == offset[i]) {
9454                                 status = -EINVAL;
9455                                 goto end;
9456                         }
9457         }
9458
9459         /* Return. */
9460         if (header)
9461                 *header = h0;
9462
9463 end:
9464         free(offset);
9465         return status;
9466 }
9467
9468 int
9469 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
9470                               const char *name,
9471                               struct rte_swx_pipeline_table_params *params,
9472                               const char *recommended_table_type_name,
9473                               const char *args,
9474                               uint32_t size)
9475 {
9476         struct table_type *type;
9477         struct table *t;
9478         struct action *default_action;
9479         struct header *header = NULL;
9480         uint32_t action_data_size_max = 0, i;
9481         int status = 0;
9482
9483         CHECK(p, EINVAL);
9484
9485         CHECK_NAME(name, EINVAL);
9486         CHECK(!table_find(p, name), EEXIST);
9487         CHECK(!selector_find(p, name), EEXIST);
9488
9489         CHECK(params, EINVAL);
9490
9491         /* Match checks. */
9492         status = table_match_fields_check(p, params, &header);
9493         if (status)
9494                 return status;
9495
9496         /* Action checks. */
9497         CHECK(params->n_actions, EINVAL);
9498         CHECK(params->action_names, EINVAL);
9499         for (i = 0; i < params->n_actions; i++) {
9500                 const char *action_name = params->action_names[i];
9501                 struct action *a;
9502                 uint32_t action_data_size;
9503
9504                 CHECK_NAME(action_name, EINVAL);
9505
9506                 a = action_find(p, action_name);
9507                 CHECK(a, EINVAL);
9508
9509                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
9510                 if (action_data_size > action_data_size_max)
9511                         action_data_size_max = action_data_size;
9512         }
9513
9514         CHECK_NAME(params->default_action_name, EINVAL);
9515         for (i = 0; i < p->n_actions; i++)
9516                 if (!strcmp(params->action_names[i],
9517                             params->default_action_name))
9518                         break;
9519         CHECK(i < params->n_actions, EINVAL);
9520         default_action = action_find(p, params->default_action_name);
9521         CHECK((default_action->st && params->default_action_data) ||
9522               !params->default_action_data, EINVAL);
9523
9524         /* Table type checks. */
9525         if (recommended_table_type_name)
9526                 CHECK_NAME(recommended_table_type_name, EINVAL);
9527
9528         if (params->n_fields) {
9529                 enum rte_swx_table_match_type match_type;
9530
9531                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
9532                 if (status)
9533                         return status;
9534
9535                 type = table_type_resolve(p, recommended_table_type_name, match_type);
9536                 CHECK(type, EINVAL);
9537         } else {
9538                 type = NULL;
9539         }
9540
9541         /* Memory allocation. */
9542         t = calloc(1, sizeof(struct table));
9543         CHECK(t, ENOMEM);
9544
9545         t->fields = calloc(params->n_fields, sizeof(struct match_field));
9546         if (!t->fields) {
9547                 free(t);
9548                 CHECK(0, ENOMEM);
9549         }
9550
9551         t->actions = calloc(params->n_actions, sizeof(struct action *));
9552         if (!t->actions) {
9553                 free(t->fields);
9554                 free(t);
9555                 CHECK(0, ENOMEM);
9556         }
9557
9558         if (action_data_size_max) {
9559                 t->default_action_data = calloc(1, action_data_size_max);
9560                 if (!t->default_action_data) {
9561                         free(t->actions);
9562                         free(t->fields);
9563                         free(t);
9564                         CHECK(0, ENOMEM);
9565                 }
9566         }
9567
9568         /* Node initialization. */
9569         strcpy(t->name, name);
9570         if (args && args[0])
9571                 strcpy(t->args, args);
9572         t->type = type;
9573
9574         for (i = 0; i < params->n_fields; i++) {
9575                 struct rte_swx_match_field_params *field = &params->fields[i];
9576                 struct match_field *f = &t->fields[i];
9577
9578                 f->match_type = field->match_type;
9579                 f->field = header ?
9580                         header_field_parse(p, field->name, NULL) :
9581                         metadata_field_parse(p, field->name);
9582         }
9583         t->n_fields = params->n_fields;
9584         t->header = header;
9585
9586         for (i = 0; i < params->n_actions; i++)
9587                 t->actions[i] = action_find(p, params->action_names[i]);
9588         t->default_action = default_action;
9589         if (default_action->st)
9590                 memcpy(t->default_action_data,
9591                        params->default_action_data,
9592                        default_action->st->n_bits / 8);
9593         t->n_actions = params->n_actions;
9594         t->default_action_is_const = params->default_action_is_const;
9595         t->action_data_size_max = action_data_size_max;
9596
9597         t->size = size;
9598         t->id = p->n_tables;
9599
9600         /* Node add to tailq. */
9601         TAILQ_INSERT_TAIL(&p->tables, t, node);
9602         p->n_tables++;
9603
9604         return 0;
9605 }
9606
9607 static struct rte_swx_table_params *
9608 table_params_get(struct table *table)
9609 {
9610         struct rte_swx_table_params *params;
9611         struct field *first, *last;
9612         uint8_t *key_mask;
9613         uint32_t key_size, key_offset, action_data_size, i;
9614
9615         /* Memory allocation. */
9616         params = calloc(1, sizeof(struct rte_swx_table_params));
9617         if (!params)
9618                 return NULL;
9619
9620         /* Find first (smallest offset) and last (biggest offset) match fields. */
9621         first = table->fields[0].field;
9622         last = table->fields[0].field;
9623
9624         for (i = 0; i < table->n_fields; i++) {
9625                 struct field *f = table->fields[i].field;
9626
9627                 if (f->offset < first->offset)
9628                         first = f;
9629
9630                 if (f->offset > last->offset)
9631                         last = f;
9632         }
9633
9634         /* Key offset and size. */
9635         key_offset = first->offset / 8;
9636         key_size = (last->offset + last->n_bits - first->offset) / 8;
9637
9638         /* Memory allocation. */
9639         key_mask = calloc(1, key_size);
9640         if (!key_mask) {
9641                 free(params);
9642                 return NULL;
9643         }
9644
9645         /* Key mask. */
9646         for (i = 0; i < table->n_fields; i++) {
9647                 struct field *f = table->fields[i].field;
9648                 uint32_t start = (f->offset - first->offset) / 8;
9649                 size_t size = f->n_bits / 8;
9650
9651                 memset(&key_mask[start], 0xFF, size);
9652         }
9653
9654         /* Action data size. */
9655         action_data_size = 0;
9656         for (i = 0; i < table->n_actions; i++) {
9657                 struct action *action = table->actions[i];
9658                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
9659
9660                 if (ads > action_data_size)
9661                         action_data_size = ads;
9662         }
9663
9664         /* Fill in. */
9665         params->match_type = table->type->match_type;
9666         params->key_size = key_size;
9667         params->key_offset = key_offset;
9668         params->key_mask0 = key_mask;
9669         params->action_data_size = action_data_size;
9670         params->n_keys_max = table->size;
9671
9672         return params;
9673 }
9674
9675 static void
9676 table_params_free(struct rte_swx_table_params *params)
9677 {
9678         if (!params)
9679                 return;
9680
9681         free(params->key_mask0);
9682         free(params);
9683 }
9684
9685 static int
9686 table_stub_lkp(void *table __rte_unused,
9687                void *mailbox __rte_unused,
9688                uint8_t **key __rte_unused,
9689                uint64_t *action_id __rte_unused,
9690                uint8_t **action_data __rte_unused,
9691                int *hit)
9692 {
9693         *hit = 0;
9694         return 1; /* DONE. */
9695 }
9696
9697 static int
9698 table_build(struct rte_swx_pipeline *p)
9699 {
9700         uint32_t i;
9701
9702         /* Per pipeline: table statistics. */
9703         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
9704         CHECK(p->table_stats, ENOMEM);
9705
9706         for (i = 0; i < p->n_tables; i++) {
9707                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9708                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
9709         }
9710
9711         /* Per thread: table runt-time. */
9712         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9713                 struct thread *t = &p->threads[i];
9714                 struct table *table;
9715
9716                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
9717                 CHECK(t->tables, ENOMEM);
9718
9719                 TAILQ_FOREACH(table, &p->tables, node) {
9720                         struct table_runtime *r = &t->tables[table->id];
9721
9722                         if (table->type) {
9723                                 uint64_t size;
9724
9725                                 size = table->type->ops.mailbox_size_get();
9726
9727                                 /* r->func. */
9728                                 r->func = table->type->ops.lkp;
9729
9730                                 /* r->mailbox. */
9731                                 if (size) {
9732                                         r->mailbox = calloc(1, size);
9733                                         CHECK(r->mailbox, ENOMEM);
9734                                 }
9735
9736                                 /* r->key. */
9737                                 r->key = table->header ?
9738                                         &t->structs[table->header->struct_id] :
9739                                         &t->structs[p->metadata_struct_id];
9740                         } else {
9741                                 r->func = table_stub_lkp;
9742                         }
9743                 }
9744         }
9745
9746         return 0;
9747 }
9748
9749 static void
9750 table_build_free(struct rte_swx_pipeline *p)
9751 {
9752         uint32_t i;
9753
9754         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9755                 struct thread *t = &p->threads[i];
9756                 uint32_t j;
9757
9758                 if (!t->tables)
9759                         continue;
9760
9761                 for (j = 0; j < p->n_tables; j++) {
9762                         struct table_runtime *r = &t->tables[j];
9763
9764                         free(r->mailbox);
9765                 }
9766
9767                 free(t->tables);
9768                 t->tables = NULL;
9769         }
9770
9771         if (p->table_stats) {
9772                 for (i = 0; i < p->n_tables; i++)
9773                         free(p->table_stats[i].n_pkts_action);
9774
9775                 free(p->table_stats);
9776         }
9777 }
9778
9779 static void
9780 table_free(struct rte_swx_pipeline *p)
9781 {
9782         table_build_free(p);
9783
9784         /* Tables. */
9785         for ( ; ; ) {
9786                 struct table *elem;
9787
9788                 elem = TAILQ_FIRST(&p->tables);
9789                 if (!elem)
9790                         break;
9791
9792                 TAILQ_REMOVE(&p->tables, elem, node);
9793                 free(elem->fields);
9794                 free(elem->actions);
9795                 free(elem->default_action_data);
9796                 free(elem);
9797         }
9798
9799         /* Table types. */
9800         for ( ; ; ) {
9801                 struct table_type *elem;
9802
9803                 elem = TAILQ_FIRST(&p->table_types);
9804                 if (!elem)
9805                         break;
9806
9807                 TAILQ_REMOVE(&p->table_types, elem, node);
9808                 free(elem);
9809         }
9810 }
9811
9812 /*
9813  * Selector.
9814  */
9815 static struct selector *
9816 selector_find(struct rte_swx_pipeline *p, const char *name)
9817 {
9818         struct selector *s;
9819
9820         TAILQ_FOREACH(s, &p->selectors, node)
9821                 if (strcmp(s->name, name) == 0)
9822                         return s;
9823
9824         return NULL;
9825 }
9826
9827 static struct selector *
9828 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9829 {
9830         struct selector *s = NULL;
9831
9832         TAILQ_FOREACH(s, &p->selectors, node)
9833                 if (s->id == id)
9834                         return s;
9835
9836         return NULL;
9837 }
9838
9839 static int
9840 selector_fields_check(struct rte_swx_pipeline *p,
9841                       struct rte_swx_pipeline_selector_params *params,
9842                       struct header **header)
9843 {
9844         struct header *h0 = NULL;
9845         struct field *hf, *mf;
9846         uint32_t i;
9847
9848         /* Return if no selector fields. */
9849         if (!params->n_selector_fields || !params->selector_field_names)
9850                 return -EINVAL;
9851
9852         /* Check that all the selector fields either belong to the same header
9853          * or are all meta-data fields.
9854          */
9855         hf = header_field_parse(p, params->selector_field_names[0], &h0);
9856         mf = metadata_field_parse(p, params->selector_field_names[0]);
9857         if (!hf && !mf)
9858                 return -EINVAL;
9859
9860         for (i = 1; i < params->n_selector_fields; i++)
9861                 if (h0) {
9862                         struct header *h;
9863
9864                         hf = header_field_parse(p, params->selector_field_names[i], &h);
9865                         if (!hf || (h->id != h0->id))
9866                                 return -EINVAL;
9867                 } else {
9868                         mf = metadata_field_parse(p, params->selector_field_names[i]);
9869                         if (!mf)
9870                                 return -EINVAL;
9871                 }
9872
9873         /* Check that there are no duplicated match fields. */
9874         for (i = 0; i < params->n_selector_fields; i++) {
9875                 const char *field_name = params->selector_field_names[i];
9876                 uint32_t j;
9877
9878                 for (j = i + 1; j < params->n_selector_fields; j++)
9879                         if (!strcmp(params->selector_field_names[j], field_name))
9880                                 return -EINVAL;
9881         }
9882
9883         /* Return. */
9884         if (header)
9885                 *header = h0;
9886
9887         return 0;
9888 }
9889
9890 int
9891 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
9892                                  const char *name,
9893                                  struct rte_swx_pipeline_selector_params *params)
9894 {
9895         struct selector *s;
9896         struct header *selector_header = NULL;
9897         struct field *group_id_field, *member_id_field;
9898         uint32_t i;
9899         int status = 0;
9900
9901         CHECK(p, EINVAL);
9902
9903         CHECK_NAME(name, EINVAL);
9904         CHECK(!table_find(p, name), EEXIST);
9905         CHECK(!selector_find(p, name), EEXIST);
9906
9907         CHECK(params, EINVAL);
9908
9909         CHECK_NAME(params->group_id_field_name, EINVAL);
9910         group_id_field = metadata_field_parse(p, params->group_id_field_name);
9911         CHECK(group_id_field, EINVAL);
9912
9913         for (i = 0; i < params->n_selector_fields; i++) {
9914                 const char *field_name = params->selector_field_names[i];
9915
9916                 CHECK_NAME(field_name, EINVAL);
9917         }
9918         status = selector_fields_check(p, params, &selector_header);
9919         if (status)
9920                 return status;
9921
9922         CHECK_NAME(params->member_id_field_name, EINVAL);
9923         member_id_field = metadata_field_parse(p, params->member_id_field_name);
9924         CHECK(member_id_field, EINVAL);
9925
9926         CHECK(params->n_groups_max, EINVAL);
9927
9928         CHECK(params->n_members_per_group_max, EINVAL);
9929
9930         /* Memory allocation. */
9931         s = calloc(1, sizeof(struct selector));
9932         if (!s) {
9933                 status = -ENOMEM;
9934                 goto error;
9935         }
9936
9937         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
9938         if (!s->selector_fields) {
9939                 status = -ENOMEM;
9940                 goto error;
9941         }
9942
9943         /* Node initialization. */
9944         strcpy(s->name, name);
9945
9946         s->group_id_field = group_id_field;
9947
9948         for (i = 0; i < params->n_selector_fields; i++) {
9949                 const char *field_name = params->selector_field_names[i];
9950
9951                 s->selector_fields[i] = selector_header ?
9952                         header_field_parse(p, field_name, NULL) :
9953                         metadata_field_parse(p, field_name);
9954         }
9955
9956         s->n_selector_fields = params->n_selector_fields;
9957
9958         s->selector_header = selector_header;
9959
9960         s->member_id_field = member_id_field;
9961
9962         s->n_groups_max = params->n_groups_max;
9963
9964         s->n_members_per_group_max = params->n_members_per_group_max;
9965
9966         s->id = p->n_selectors;
9967
9968         /* Node add to tailq. */
9969         TAILQ_INSERT_TAIL(&p->selectors, s, node);
9970         p->n_selectors++;
9971
9972         return 0;
9973
9974 error:
9975         if (!s)
9976                 return status;
9977
9978         free(s->selector_fields);
9979
9980         free(s);
9981
9982         return status;
9983 }
9984
9985 static void
9986 selector_params_free(struct rte_swx_table_selector_params *params)
9987 {
9988         if (!params)
9989                 return;
9990
9991         free(params->selector_mask);
9992
9993         free(params);
9994 }
9995
9996 static struct rte_swx_table_selector_params *
9997 selector_table_params_get(struct selector *s)
9998 {
9999         struct rte_swx_table_selector_params *params = NULL;
10000         struct field *first, *last;
10001         uint32_t i;
10002
10003         /* Memory allocation. */
10004         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
10005         if (!params)
10006                 goto error;
10007
10008         /* Group ID. */
10009         params->group_id_offset = s->group_id_field->offset / 8;
10010
10011         /* Find first (smallest offset) and last (biggest offset) selector fields. */
10012         first = s->selector_fields[0];
10013         last = s->selector_fields[0];
10014
10015         for (i = 0; i < s->n_selector_fields; i++) {
10016                 struct field *f = s->selector_fields[i];
10017
10018                 if (f->offset < first->offset)
10019                         first = f;
10020
10021                 if (f->offset > last->offset)
10022                         last = f;
10023         }
10024
10025         /* Selector offset and size. */
10026         params->selector_offset = first->offset / 8;
10027         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
10028
10029         /* Memory allocation. */
10030         params->selector_mask = calloc(1, params->selector_size);
10031         if (!params->selector_mask)
10032                 goto error;
10033
10034         /* Selector mask. */
10035         for (i = 0; i < s->n_selector_fields; i++) {
10036                 struct field *f = s->selector_fields[i];
10037                 uint32_t start = (f->offset - first->offset) / 8;
10038                 size_t size = f->n_bits / 8;
10039
10040                 memset(&params->selector_mask[start], 0xFF, size);
10041         }
10042
10043         /* Member ID. */
10044         params->member_id_offset = s->member_id_field->offset / 8;
10045
10046         /* Maximum number of groups. */
10047         params->n_groups_max = s->n_groups_max;
10048
10049         /* Maximum number of members per group. */
10050         params->n_members_per_group_max = s->n_members_per_group_max;
10051
10052         return params;
10053
10054 error:
10055         selector_params_free(params);
10056         return NULL;
10057 }
10058
10059 static void
10060 selector_build_free(struct rte_swx_pipeline *p)
10061 {
10062         uint32_t i;
10063
10064         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10065                 struct thread *t = &p->threads[i];
10066                 uint32_t j;
10067
10068                 if (!t->selectors)
10069                         continue;
10070
10071                 for (j = 0; j < p->n_selectors; j++) {
10072                         struct selector_runtime *r = &t->selectors[j];
10073
10074                         free(r->mailbox);
10075                 }
10076
10077                 free(t->selectors);
10078                 t->selectors = NULL;
10079         }
10080
10081         free(p->selector_stats);
10082         p->selector_stats = NULL;
10083 }
10084
10085 static int
10086 selector_build(struct rte_swx_pipeline *p)
10087 {
10088         uint32_t i;
10089         int status = 0;
10090
10091         /* Per pipeline: selector statistics. */
10092         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
10093         if (!p->selector_stats) {
10094                 status = -ENOMEM;
10095                 goto error;
10096         }
10097
10098         /* Per thread: selector run-time. */
10099         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10100                 struct thread *t = &p->threads[i];
10101                 struct selector *s;
10102
10103                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
10104                 if (!t->selectors) {
10105                         status = -ENOMEM;
10106                         goto error;
10107                 }
10108
10109                 TAILQ_FOREACH(s, &p->selectors, node) {
10110                         struct selector_runtime *r = &t->selectors[s->id];
10111                         uint64_t size;
10112
10113                         /* r->mailbox. */
10114                         size = rte_swx_table_selector_mailbox_size_get();
10115                         if (size) {
10116                                 r->mailbox = calloc(1, size);
10117                                 if (!r->mailbox) {
10118                                         status = -ENOMEM;
10119                                         goto error;
10120                                 }
10121                         }
10122
10123                         /* r->group_id_buffer. */
10124                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
10125
10126                         /* r->selector_buffer. */
10127                         r->selector_buffer = s->selector_header ?
10128                                 &t->structs[s->selector_header->struct_id] :
10129                                 &t->structs[p->metadata_struct_id];
10130
10131                         /* r->member_id_buffer. */
10132                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
10133                 }
10134         }
10135
10136         return 0;
10137
10138 error:
10139         selector_build_free(p);
10140         return status;
10141 }
10142
10143 static void
10144 selector_free(struct rte_swx_pipeline *p)
10145 {
10146         selector_build_free(p);
10147
10148         /* Selector tables. */
10149         for ( ; ; ) {
10150                 struct selector *elem;
10151
10152                 elem = TAILQ_FIRST(&p->selectors);
10153                 if (!elem)
10154                         break;
10155
10156                 TAILQ_REMOVE(&p->selectors, elem, node);
10157                 free(elem->selector_fields);
10158                 free(elem);
10159         }
10160 }
10161
10162 /*
10163  * Table state.
10164  */
10165 static int
10166 table_state_build(struct rte_swx_pipeline *p)
10167 {
10168         struct table *table;
10169         struct selector *s;
10170
10171         p->table_state = calloc(p->n_tables + p->n_selectors,
10172                                 sizeof(struct rte_swx_table_state));
10173         CHECK(p->table_state, ENOMEM);
10174
10175         TAILQ_FOREACH(table, &p->tables, node) {
10176                 struct rte_swx_table_state *ts = &p->table_state[table->id];
10177
10178                 if (table->type) {
10179                         struct rte_swx_table_params *params;
10180
10181                         /* ts->obj. */
10182                         params = table_params_get(table);
10183                         CHECK(params, ENOMEM);
10184
10185                         ts->obj = table->type->ops.create(params,
10186                                 NULL,
10187                                 table->args,
10188                                 p->numa_node);
10189
10190                         table_params_free(params);
10191                         CHECK(ts->obj, ENODEV);
10192                 }
10193
10194                 /* ts->default_action_data. */
10195                 if (table->action_data_size_max) {
10196                         ts->default_action_data =
10197                                 malloc(table->action_data_size_max);
10198                         CHECK(ts->default_action_data, ENOMEM);
10199
10200                         memcpy(ts->default_action_data,
10201                                table->default_action_data,
10202                                table->action_data_size_max);
10203                 }
10204
10205                 /* ts->default_action_id. */
10206                 ts->default_action_id = table->default_action->id;
10207         }
10208
10209         TAILQ_FOREACH(s, &p->selectors, node) {
10210                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
10211                 struct rte_swx_table_selector_params *params;
10212
10213                 /* ts->obj. */
10214                 params = selector_table_params_get(s);
10215                 CHECK(params, ENOMEM);
10216
10217                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
10218
10219                 selector_params_free(params);
10220                 CHECK(ts->obj, ENODEV);
10221         }
10222
10223         return 0;
10224 }
10225
10226 static void
10227 table_state_build_free(struct rte_swx_pipeline *p)
10228 {
10229         uint32_t i;
10230
10231         if (!p->table_state)
10232                 return;
10233
10234         for (i = 0; i < p->n_tables; i++) {
10235                 struct rte_swx_table_state *ts = &p->table_state[i];
10236                 struct table *table = table_find_by_id(p, i);
10237
10238                 /* ts->obj. */
10239                 if (table->type && ts->obj)
10240                         table->type->ops.free(ts->obj);
10241
10242                 /* ts->default_action_data. */
10243                 free(ts->default_action_data);
10244         }
10245
10246         for (i = 0; i < p->n_selectors; i++) {
10247                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
10248
10249                 /* ts->obj. */
10250                 if (ts->obj)
10251                         rte_swx_table_selector_free(ts->obj);
10252         }
10253
10254         free(p->table_state);
10255         p->table_state = NULL;
10256 }
10257
10258 static void
10259 table_state_free(struct rte_swx_pipeline *p)
10260 {
10261         table_state_build_free(p);
10262 }
10263
10264 /*
10265  * Register array.
10266  */
10267 static struct regarray *
10268 regarray_find(struct rte_swx_pipeline *p, const char *name)
10269 {
10270         struct regarray *elem;
10271
10272         TAILQ_FOREACH(elem, &p->regarrays, node)
10273                 if (!strcmp(elem->name, name))
10274                         return elem;
10275
10276         return NULL;
10277 }
10278
10279 static struct regarray *
10280 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
10281 {
10282         struct regarray *elem = NULL;
10283
10284         TAILQ_FOREACH(elem, &p->regarrays, node)
10285                 if (elem->id == id)
10286                         return elem;
10287
10288         return NULL;
10289 }
10290
10291 int
10292 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
10293                               const char *name,
10294                               uint32_t size,
10295                               uint64_t init_val)
10296 {
10297         struct regarray *r;
10298
10299         CHECK(p, EINVAL);
10300
10301         CHECK_NAME(name, EINVAL);
10302         CHECK(!regarray_find(p, name), EEXIST);
10303
10304         CHECK(size, EINVAL);
10305         size = rte_align32pow2(size);
10306
10307         /* Memory allocation. */
10308         r = calloc(1, sizeof(struct regarray));
10309         CHECK(r, ENOMEM);
10310
10311         /* Node initialization. */
10312         strcpy(r->name, name);
10313         r->init_val = init_val;
10314         r->size = size;
10315         r->id = p->n_regarrays;
10316
10317         /* Node add to tailq. */
10318         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
10319         p->n_regarrays++;
10320
10321         return 0;
10322 }
10323
10324 static int
10325 regarray_build(struct rte_swx_pipeline *p)
10326 {
10327         struct regarray *regarray;
10328
10329         if (!p->n_regarrays)
10330                 return 0;
10331
10332         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
10333         CHECK(p->regarray_runtime, ENOMEM);
10334
10335         TAILQ_FOREACH(regarray, &p->regarrays, node) {
10336                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
10337                 uint32_t i;
10338
10339                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
10340                                          RTE_CACHE_LINE_SIZE,
10341                                          p->numa_node);
10342                 CHECK(r->regarray, ENOMEM);
10343
10344                 if (regarray->init_val)
10345                         for (i = 0; i < regarray->size; i++)
10346                                 r->regarray[i] = regarray->init_val;
10347
10348                 r->size_mask = regarray->size - 1;
10349         }
10350
10351         return 0;
10352 }
10353
10354 static void
10355 regarray_build_free(struct rte_swx_pipeline *p)
10356 {
10357         uint32_t i;
10358
10359         if (!p->regarray_runtime)
10360                 return;
10361
10362         for (i = 0; i < p->n_regarrays; i++) {
10363                 struct regarray *regarray = regarray_find_by_id(p, i);
10364                 struct regarray_runtime *r = &p->regarray_runtime[i];
10365
10366                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
10367         }
10368
10369         free(p->regarray_runtime);
10370         p->regarray_runtime = NULL;
10371 }
10372
10373 static void
10374 regarray_free(struct rte_swx_pipeline *p)
10375 {
10376         regarray_build_free(p);
10377
10378         for ( ; ; ) {
10379                 struct regarray *elem;
10380
10381                 elem = TAILQ_FIRST(&p->regarrays);
10382                 if (!elem)
10383                         break;
10384
10385                 TAILQ_REMOVE(&p->regarrays, elem, node);
10386                 free(elem);
10387         }
10388 }
10389
10390 /*
10391  * Meter array.
10392  */
10393 static struct meter_profile *
10394 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
10395 {
10396         struct meter_profile *elem;
10397
10398         TAILQ_FOREACH(elem, &p->meter_profiles, node)
10399                 if (!strcmp(elem->name, name))
10400                         return elem;
10401
10402         return NULL;
10403 }
10404
10405 static struct metarray *
10406 metarray_find(struct rte_swx_pipeline *p, const char *name)
10407 {
10408         struct metarray *elem;
10409
10410         TAILQ_FOREACH(elem, &p->metarrays, node)
10411                 if (!strcmp(elem->name, name))
10412                         return elem;
10413
10414         return NULL;
10415 }
10416
10417 static struct metarray *
10418 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
10419 {
10420         struct metarray *elem = NULL;
10421
10422         TAILQ_FOREACH(elem, &p->metarrays, node)
10423                 if (elem->id == id)
10424                         return elem;
10425
10426         return NULL;
10427 }
10428
10429 int
10430 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
10431                                  const char *name,
10432                                  uint32_t size)
10433 {
10434         struct metarray *m;
10435
10436         CHECK(p, EINVAL);
10437
10438         CHECK_NAME(name, EINVAL);
10439         CHECK(!metarray_find(p, name), EEXIST);
10440
10441         CHECK(size, EINVAL);
10442         size = rte_align32pow2(size);
10443
10444         /* Memory allocation. */
10445         m = calloc(1, sizeof(struct metarray));
10446         CHECK(m, ENOMEM);
10447
10448         /* Node initialization. */
10449         strcpy(m->name, name);
10450         m->size = size;
10451         m->id = p->n_metarrays;
10452
10453         /* Node add to tailq. */
10454         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
10455         p->n_metarrays++;
10456
10457         return 0;
10458 }
10459
10460 struct meter_profile meter_profile_default = {
10461         .node = {0},
10462         .name = "",
10463         .params = {0},
10464
10465         .profile = {
10466                 .cbs = 10000,
10467                 .pbs = 10000,
10468                 .cir_period = 1,
10469                 .cir_bytes_per_period = 1,
10470                 .pir_period = 1,
10471                 .pir_bytes_per_period = 1,
10472         },
10473
10474         .n_users = 0,
10475 };
10476
10477 static void
10478 meter_init(struct meter *m)
10479 {
10480         memset(m, 0, sizeof(struct meter));
10481         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
10482         m->profile = &meter_profile_default;
10483         m->color_mask = RTE_COLOR_GREEN;
10484
10485         meter_profile_default.n_users++;
10486 }
10487
10488 static int
10489 metarray_build(struct rte_swx_pipeline *p)
10490 {
10491         struct metarray *m;
10492
10493         if (!p->n_metarrays)
10494                 return 0;
10495
10496         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
10497         CHECK(p->metarray_runtime, ENOMEM);
10498
10499         TAILQ_FOREACH(m, &p->metarrays, node) {
10500                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
10501                 uint32_t i;
10502
10503                 r->metarray = env_malloc(m->size * sizeof(struct meter),
10504                                          RTE_CACHE_LINE_SIZE,
10505                                          p->numa_node);
10506                 CHECK(r->metarray, ENOMEM);
10507
10508                 for (i = 0; i < m->size; i++)
10509                         meter_init(&r->metarray[i]);
10510
10511                 r->size_mask = m->size - 1;
10512         }
10513
10514         return 0;
10515 }
10516
10517 static void
10518 metarray_build_free(struct rte_swx_pipeline *p)
10519 {
10520         uint32_t i;
10521
10522         if (!p->metarray_runtime)
10523                 return;
10524
10525         for (i = 0; i < p->n_metarrays; i++) {
10526                 struct metarray *m = metarray_find_by_id(p, i);
10527                 struct metarray_runtime *r = &p->metarray_runtime[i];
10528
10529                 env_free(r->metarray, m->size * sizeof(struct meter));
10530         }
10531
10532         free(p->metarray_runtime);
10533         p->metarray_runtime = NULL;
10534 }
10535
10536 static void
10537 metarray_free(struct rte_swx_pipeline *p)
10538 {
10539         metarray_build_free(p);
10540
10541         /* Meter arrays. */
10542         for ( ; ; ) {
10543                 struct metarray *elem;
10544
10545                 elem = TAILQ_FIRST(&p->metarrays);
10546                 if (!elem)
10547                         break;
10548
10549                 TAILQ_REMOVE(&p->metarrays, elem, node);
10550                 free(elem);
10551         }
10552
10553         /* Meter profiles. */
10554         for ( ; ; ) {
10555                 struct meter_profile *elem;
10556
10557                 elem = TAILQ_FIRST(&p->meter_profiles);
10558                 if (!elem)
10559                         break;
10560
10561                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
10562                 free(elem);
10563         }
10564 }
10565
10566 /*
10567  * Pipeline.
10568  */
10569 int
10570 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
10571 {
10572         struct rte_swx_pipeline *pipeline;
10573
10574         /* Check input parameters. */
10575         CHECK(p, EINVAL);
10576
10577         /* Memory allocation. */
10578         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
10579         CHECK(pipeline, ENOMEM);
10580
10581         /* Initialization. */
10582         TAILQ_INIT(&pipeline->struct_types);
10583         TAILQ_INIT(&pipeline->port_in_types);
10584         TAILQ_INIT(&pipeline->ports_in);
10585         TAILQ_INIT(&pipeline->port_out_types);
10586         TAILQ_INIT(&pipeline->ports_out);
10587         TAILQ_INIT(&pipeline->extern_types);
10588         TAILQ_INIT(&pipeline->extern_objs);
10589         TAILQ_INIT(&pipeline->extern_funcs);
10590         TAILQ_INIT(&pipeline->headers);
10591         TAILQ_INIT(&pipeline->actions);
10592         TAILQ_INIT(&pipeline->table_types);
10593         TAILQ_INIT(&pipeline->tables);
10594         TAILQ_INIT(&pipeline->selectors);
10595         TAILQ_INIT(&pipeline->regarrays);
10596         TAILQ_INIT(&pipeline->meter_profiles);
10597         TAILQ_INIT(&pipeline->metarrays);
10598
10599         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
10600         pipeline->numa_node = numa_node;
10601
10602         *p = pipeline;
10603         return 0;
10604 }
10605
10606 void
10607 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
10608 {
10609         if (!p)
10610                 return;
10611
10612         free(p->instructions);
10613
10614         metarray_free(p);
10615         regarray_free(p);
10616         table_state_free(p);
10617         selector_free(p);
10618         table_free(p);
10619         action_free(p);
10620         metadata_free(p);
10621         header_free(p);
10622         extern_func_free(p);
10623         extern_obj_free(p);
10624         port_out_free(p);
10625         port_in_free(p);
10626         struct_free(p);
10627
10628         free(p);
10629 }
10630
10631 int
10632 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
10633                                      const char **instructions,
10634                                      uint32_t n_instructions)
10635 {
10636         int err;
10637         uint32_t i;
10638
10639         err = instruction_config(p, NULL, instructions, n_instructions);
10640         if (err)
10641                 return err;
10642
10643         /* Thread instruction pointer reset. */
10644         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10645                 struct thread *t = &p->threads[i];
10646
10647                 thread_ip_reset(p, t);
10648         }
10649
10650         return 0;
10651 }
10652
10653 int
10654 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
10655 {
10656         int status;
10657
10658         CHECK(p, EINVAL);
10659         CHECK(p->build_done == 0, EEXIST);
10660
10661         status = port_in_build(p);
10662         if (status)
10663                 goto error;
10664
10665         status = port_out_build(p);
10666         if (status)
10667                 goto error;
10668
10669         status = struct_build(p);
10670         if (status)
10671                 goto error;
10672
10673         status = extern_obj_build(p);
10674         if (status)
10675                 goto error;
10676
10677         status = extern_func_build(p);
10678         if (status)
10679                 goto error;
10680
10681         status = header_build(p);
10682         if (status)
10683                 goto error;
10684
10685         status = metadata_build(p);
10686         if (status)
10687                 goto error;
10688
10689         status = action_build(p);
10690         if (status)
10691                 goto error;
10692
10693         status = table_build(p);
10694         if (status)
10695                 goto error;
10696
10697         status = selector_build(p);
10698         if (status)
10699                 goto error;
10700
10701         status = table_state_build(p);
10702         if (status)
10703                 goto error;
10704
10705         status = regarray_build(p);
10706         if (status)
10707                 goto error;
10708
10709         status = metarray_build(p);
10710         if (status)
10711                 goto error;
10712
10713         p->build_done = 1;
10714         return 0;
10715
10716 error:
10717         metarray_build_free(p);
10718         regarray_build_free(p);
10719         table_state_build_free(p);
10720         selector_build_free(p);
10721         table_build_free(p);
10722         action_build_free(p);
10723         metadata_build_free(p);
10724         header_build_free(p);
10725         extern_func_build_free(p);
10726         extern_obj_build_free(p);
10727         port_out_build_free(p);
10728         port_in_build_free(p);
10729         struct_build_free(p);
10730
10731         return status;
10732 }
10733
10734 void
10735 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
10736 {
10737         uint32_t i;
10738
10739         for (i = 0; i < n_instructions; i++)
10740                 instr_exec(p);
10741 }
10742
10743 void
10744 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
10745 {
10746         uint32_t i;
10747
10748         for (i = 0; i < p->n_ports_out; i++) {
10749                 struct port_out_runtime *port = &p->out[i];
10750
10751                 if (port->flush)
10752                         port->flush(port->obj);
10753         }
10754 }
10755
10756 /*
10757  * Control.
10758  */
10759 int
10760 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
10761                               struct rte_swx_ctl_pipeline_info *pipeline)
10762 {
10763         struct action *action;
10764         struct table *table;
10765         uint32_t n_actions = 0, n_tables = 0;
10766
10767         if (!p || !pipeline)
10768                 return -EINVAL;
10769
10770         TAILQ_FOREACH(action, &p->actions, node)
10771                 n_actions++;
10772
10773         TAILQ_FOREACH(table, &p->tables, node)
10774                 n_tables++;
10775
10776         pipeline->n_ports_in = p->n_ports_in;
10777         pipeline->n_ports_out = p->n_ports_out;
10778         pipeline->n_actions = n_actions;
10779         pipeline->n_tables = n_tables;
10780         pipeline->n_selectors = p->n_selectors;
10781         pipeline->n_regarrays = p->n_regarrays;
10782         pipeline->n_metarrays = p->n_metarrays;
10783
10784         return 0;
10785 }
10786
10787 int
10788 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
10789 {
10790         if (!p || !numa_node)
10791                 return -EINVAL;
10792
10793         *numa_node = p->numa_node;
10794         return 0;
10795 }
10796
10797 int
10798 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
10799                             uint32_t action_id,
10800                             struct rte_swx_ctl_action_info *action)
10801 {
10802         struct action *a = NULL;
10803
10804         if (!p || (action_id >= p->n_actions) || !action)
10805                 return -EINVAL;
10806
10807         a = action_find_by_id(p, action_id);
10808         if (!a)
10809                 return -EINVAL;
10810
10811         strcpy(action->name, a->name);
10812         action->n_args = a->st ? a->st->n_fields : 0;
10813         return 0;
10814 }
10815
10816 int
10817 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10818                                 uint32_t action_id,
10819                                 uint32_t action_arg_id,
10820                                 struct rte_swx_ctl_action_arg_info *action_arg)
10821 {
10822         struct action *a = NULL;
10823         struct field *arg = NULL;
10824
10825         if (!p || (action_id >= p->n_actions) || !action_arg)
10826                 return -EINVAL;
10827
10828         a = action_find_by_id(p, action_id);
10829         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10830                 return -EINVAL;
10831
10832         arg = &a->st->fields[action_arg_id];
10833         strcpy(action_arg->name, arg->name);
10834         action_arg->n_bits = arg->n_bits;
10835         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10836
10837         return 0;
10838 }
10839
10840 int
10841 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10842                            uint32_t table_id,
10843                            struct rte_swx_ctl_table_info *table)
10844 {
10845         struct table *t = NULL;
10846
10847         if (!p || !table)
10848                 return -EINVAL;
10849
10850         t = table_find_by_id(p, table_id);
10851         if (!t)
10852                 return -EINVAL;
10853
10854         strcpy(table->name, t->name);
10855         strcpy(table->args, t->args);
10856         table->n_match_fields = t->n_fields;
10857         table->n_actions = t->n_actions;
10858         table->default_action_is_const = t->default_action_is_const;
10859         table->size = t->size;
10860         return 0;
10861 }
10862
10863 int
10864 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10865         uint32_t table_id,
10866         uint32_t match_field_id,
10867         struct rte_swx_ctl_table_match_field_info *match_field)
10868 {
10869         struct table *t;
10870         struct match_field *f;
10871
10872         if (!p || (table_id >= p->n_tables) || !match_field)
10873                 return -EINVAL;
10874
10875         t = table_find_by_id(p, table_id);
10876         if (!t || (match_field_id >= t->n_fields))
10877                 return -EINVAL;
10878
10879         f = &t->fields[match_field_id];
10880         match_field->match_type = f->match_type;
10881         match_field->is_header = t->header ? 1 : 0;
10882         match_field->n_bits = f->field->n_bits;
10883         match_field->offset = f->field->offset;
10884
10885         return 0;
10886 }
10887
10888 int
10889 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10890         uint32_t table_id,
10891         uint32_t table_action_id,
10892         struct rte_swx_ctl_table_action_info *table_action)
10893 {
10894         struct table *t;
10895
10896         if (!p || (table_id >= p->n_tables) || !table_action)
10897                 return -EINVAL;
10898
10899         t = table_find_by_id(p, table_id);
10900         if (!t || (table_action_id >= t->n_actions))
10901                 return -EINVAL;
10902
10903         table_action->action_id = t->actions[table_action_id]->id;
10904
10905         return 0;
10906 }
10907
10908 int
10909 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10910                           uint32_t table_id,
10911                           struct rte_swx_table_ops *table_ops,
10912                           int *is_stub)
10913 {
10914         struct table *t;
10915
10916         if (!p || (table_id >= p->n_tables))
10917                 return -EINVAL;
10918
10919         t = table_find_by_id(p, table_id);
10920         if (!t)
10921                 return -EINVAL;
10922
10923         if (t->type) {
10924                 if (table_ops)
10925                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10926                 *is_stub = 0;
10927         } else {
10928                 *is_stub = 1;
10929         }
10930
10931         return 0;
10932 }
10933
10934 int
10935 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10936                               uint32_t selector_id,
10937                               struct rte_swx_ctl_selector_info *selector)
10938 {
10939         struct selector *s = NULL;
10940
10941         if (!p || !selector)
10942                 return -EINVAL;
10943
10944         s = selector_find_by_id(p, selector_id);
10945         if (!s)
10946                 return -EINVAL;
10947
10948         strcpy(selector->name, s->name);
10949
10950         selector->n_selector_fields = s->n_selector_fields;
10951         selector->n_groups_max = s->n_groups_max;
10952         selector->n_members_per_group_max = s->n_members_per_group_max;
10953
10954         return 0;
10955 }
10956
10957 int
10958 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10959          uint32_t selector_id,
10960          struct rte_swx_ctl_table_match_field_info *field)
10961 {
10962         struct selector *s;
10963
10964         if (!p || (selector_id >= p->n_selectors) || !field)
10965                 return -EINVAL;
10966
10967         s = selector_find_by_id(p, selector_id);
10968         if (!s)
10969                 return -EINVAL;
10970
10971         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10972         field->is_header = 0;
10973         field->n_bits = s->group_id_field->n_bits;
10974         field->offset = s->group_id_field->offset;
10975
10976         return 0;
10977 }
10978
10979 int
10980 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10981          uint32_t selector_id,
10982          uint32_t selector_field_id,
10983          struct rte_swx_ctl_table_match_field_info *field)
10984 {
10985         struct selector *s;
10986         struct field *f;
10987
10988         if (!p || (selector_id >= p->n_selectors) || !field)
10989                 return -EINVAL;
10990
10991         s = selector_find_by_id(p, selector_id);
10992         if (!s || (selector_field_id >= s->n_selector_fields))
10993                 return -EINVAL;
10994
10995         f = s->selector_fields[selector_field_id];
10996         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10997         field->is_header = s->selector_header ? 1 : 0;
10998         field->n_bits = f->n_bits;
10999         field->offset = f->offset;
11000
11001         return 0;
11002 }
11003
11004 int
11005 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
11006          uint32_t selector_id,
11007          struct rte_swx_ctl_table_match_field_info *field)
11008 {
11009         struct selector *s;
11010
11011         if (!p || (selector_id >= p->n_selectors) || !field)
11012                 return -EINVAL;
11013
11014         s = selector_find_by_id(p, selector_id);
11015         if (!s)
11016                 return -EINVAL;
11017
11018         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
11019         field->is_header = 0;
11020         field->n_bits = s->member_id_field->n_bits;
11021         field->offset = s->member_id_field->offset;
11022
11023         return 0;
11024 }
11025
11026 int
11027 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
11028                                  struct rte_swx_table_state **table_state)
11029 {
11030         if (!p || !table_state || !p->build_done)
11031                 return -EINVAL;
11032
11033         *table_state = p->table_state;
11034         return 0;
11035 }
11036
11037 int
11038 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
11039                                  struct rte_swx_table_state *table_state)
11040 {
11041         if (!p || !table_state || !p->build_done)
11042                 return -EINVAL;
11043
11044         p->table_state = table_state;
11045         return 0;
11046 }
11047
11048 int
11049 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
11050                                         uint32_t port_id,
11051                                         struct rte_swx_port_in_stats *stats)
11052 {
11053         struct port_in *port;
11054
11055         if (!p || !stats)
11056                 return -EINVAL;
11057
11058         port = port_in_find(p, port_id);
11059         if (!port)
11060                 return -EINVAL;
11061
11062         port->type->ops.stats_read(port->obj, stats);
11063         return 0;
11064 }
11065
11066 int
11067 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
11068                                          uint32_t port_id,
11069                                          struct rte_swx_port_out_stats *stats)
11070 {
11071         struct port_out *port;
11072
11073         if (!p || !stats)
11074                 return -EINVAL;
11075
11076         port = port_out_find(p, port_id);
11077         if (!port)
11078                 return -EINVAL;
11079
11080         port->type->ops.stats_read(port->obj, stats);
11081         return 0;
11082 }
11083
11084 int
11085 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
11086                                       const char *table_name,
11087                                       struct rte_swx_table_stats *stats)
11088 {
11089         struct table *table;
11090         struct table_statistics *table_stats;
11091
11092         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
11093                 return -EINVAL;
11094
11095         table = table_find(p, table_name);
11096         if (!table)
11097                 return -EINVAL;
11098
11099         table_stats = &p->table_stats[table->id];
11100
11101         memcpy(stats->n_pkts_action,
11102                table_stats->n_pkts_action,
11103                p->n_actions * sizeof(uint64_t));
11104
11105         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
11106         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
11107
11108         return 0;
11109 }
11110
11111 int
11112 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
11113         const char *selector_name,
11114         struct rte_swx_pipeline_selector_stats *stats)
11115 {
11116         struct selector *s;
11117
11118         if (!p || !selector_name || !selector_name[0] || !stats)
11119                 return -EINVAL;
11120
11121         s = selector_find(p, selector_name);
11122         if (!s)
11123                 return -EINVAL;
11124
11125         stats->n_pkts = p->selector_stats[s->id].n_pkts;
11126
11127         return 0;
11128 }
11129
11130 int
11131 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
11132                               uint32_t regarray_id,
11133                               struct rte_swx_ctl_regarray_info *regarray)
11134 {
11135         struct regarray *r;
11136
11137         if (!p || !regarray)
11138                 return -EINVAL;
11139
11140         r = regarray_find_by_id(p, regarray_id);
11141         if (!r)
11142                 return -EINVAL;
11143
11144         strcpy(regarray->name, r->name);
11145         regarray->size = r->size;
11146         return 0;
11147 }
11148
11149 int
11150 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
11151                                    const char *regarray_name,
11152                                    uint32_t regarray_index,
11153                                    uint64_t *value)
11154 {
11155         struct regarray *regarray;
11156         struct regarray_runtime *r;
11157
11158         if (!p || !regarray_name || !value)
11159                 return -EINVAL;
11160
11161         regarray = regarray_find(p, regarray_name);
11162         if (!regarray || (regarray_index >= regarray->size))
11163                 return -EINVAL;
11164
11165         r = &p->regarray_runtime[regarray->id];
11166         *value = r->regarray[regarray_index];
11167         return 0;
11168 }
11169
11170 int
11171 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
11172                                    const char *regarray_name,
11173                                    uint32_t regarray_index,
11174                                    uint64_t value)
11175 {
11176         struct regarray *regarray;
11177         struct regarray_runtime *r;
11178
11179         if (!p || !regarray_name)
11180                 return -EINVAL;
11181
11182         regarray = regarray_find(p, regarray_name);
11183         if (!regarray || (regarray_index >= regarray->size))
11184                 return -EINVAL;
11185
11186         r = &p->regarray_runtime[regarray->id];
11187         r->regarray[regarray_index] = value;
11188         return 0;
11189 }
11190
11191 int
11192 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
11193                               uint32_t metarray_id,
11194                               struct rte_swx_ctl_metarray_info *metarray)
11195 {
11196         struct metarray *m;
11197
11198         if (!p || !metarray)
11199                 return -EINVAL;
11200
11201         m = metarray_find_by_id(p, metarray_id);
11202         if (!m)
11203                 return -EINVAL;
11204
11205         strcpy(metarray->name, m->name);
11206         metarray->size = m->size;
11207         return 0;
11208 }
11209
11210 int
11211 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
11212                               const char *name,
11213                               struct rte_meter_trtcm_params *params)
11214 {
11215         struct meter_profile *mp;
11216         int status;
11217
11218         CHECK(p, EINVAL);
11219         CHECK_NAME(name, EINVAL);
11220         CHECK(params, EINVAL);
11221         CHECK(!meter_profile_find(p, name), EEXIST);
11222
11223         /* Node allocation. */
11224         mp = calloc(1, sizeof(struct meter_profile));
11225         CHECK(mp, ENOMEM);
11226
11227         /* Node initialization. */
11228         strcpy(mp->name, name);
11229         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
11230         status = rte_meter_trtcm_profile_config(&mp->profile, params);
11231         if (status) {
11232                 free(mp);
11233                 CHECK(0, EINVAL);
11234         }
11235
11236         /* Node add to tailq. */
11237         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
11238
11239         return 0;
11240 }
11241
11242 int
11243 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
11244                                  const char *name)
11245 {
11246         struct meter_profile *mp;
11247
11248         CHECK(p, EINVAL);
11249         CHECK_NAME(name, EINVAL);
11250
11251         mp = meter_profile_find(p, name);
11252         CHECK(mp, EINVAL);
11253         CHECK(!mp->n_users, EBUSY);
11254
11255         /* Remove node from tailq. */
11256         TAILQ_REMOVE(&p->meter_profiles, mp, node);
11257         free(mp);
11258
11259         return 0;
11260 }
11261
11262 int
11263 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
11264                         const char *metarray_name,
11265                         uint32_t metarray_index)
11266 {
11267         struct meter_profile *mp_old;
11268         struct metarray *metarray;
11269         struct metarray_runtime *metarray_runtime;
11270         struct meter *m;
11271
11272         CHECK(p, EINVAL);
11273         CHECK_NAME(metarray_name, EINVAL);
11274
11275         metarray = metarray_find(p, metarray_name);
11276         CHECK(metarray, EINVAL);
11277         CHECK(metarray_index < metarray->size, EINVAL);
11278
11279         metarray_runtime = &p->metarray_runtime[metarray->id];
11280         m = &metarray_runtime->metarray[metarray_index];
11281         mp_old = m->profile;
11282
11283         meter_init(m);
11284
11285         mp_old->n_users--;
11286
11287         return 0;
11288 }
11289
11290 int
11291 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
11292                       const char *metarray_name,
11293                       uint32_t metarray_index,
11294                       const char *profile_name)
11295 {
11296         struct meter_profile *mp, *mp_old;
11297         struct metarray *metarray;
11298         struct metarray_runtime *metarray_runtime;
11299         struct meter *m;
11300
11301         CHECK(p, EINVAL);
11302         CHECK_NAME(metarray_name, EINVAL);
11303
11304         metarray = metarray_find(p, metarray_name);
11305         CHECK(metarray, EINVAL);
11306         CHECK(metarray_index < metarray->size, EINVAL);
11307
11308         mp = meter_profile_find(p, profile_name);
11309         CHECK(mp, EINVAL);
11310
11311         metarray_runtime = &p->metarray_runtime[metarray->id];
11312         m = &metarray_runtime->metarray[metarray_index];
11313         mp_old = m->profile;
11314
11315         memset(m, 0, sizeof(struct meter));
11316         rte_meter_trtcm_config(&m->m, &mp->profile);
11317         m->profile = mp;
11318         m->color_mask = RTE_COLORS;
11319
11320         mp->n_users++;
11321         mp_old->n_users--;
11322
11323         return 0;
11324 }
11325
11326 int
11327 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
11328                              const char *metarray_name,
11329                              uint32_t metarray_index,
11330                              struct rte_swx_ctl_meter_stats *stats)
11331 {
11332         struct metarray *metarray;
11333         struct metarray_runtime *metarray_runtime;
11334         struct meter *m;
11335
11336         CHECK(p, EINVAL);
11337         CHECK_NAME(metarray_name, EINVAL);
11338
11339         metarray = metarray_find(p, metarray_name);
11340         CHECK(metarray, EINVAL);
11341         CHECK(metarray_index < metarray->size, EINVAL);
11342
11343         CHECK(stats, EINVAL);
11344
11345         metarray_runtime = &p->metarray_runtime[metarray->id];
11346         m = &metarray_runtime->metarray[metarray_index];
11347
11348         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
11349         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
11350
11351         return 0;
11352 }