lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <inttypes.h>
   9 #include <sys/queue.h>
  10 #include <arpa/inet.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_prefetch.h>
  14 #include <rte_byteorder.h>
  15 #include <rte_cycles.h>
  16 #include <rte_meter.h>
  17
  18 #include <rte_swx_table_selector.h>
  19
  20 #include "rte_swx_pipeline.h"
  21 #include "rte_swx_ctl.h"
  22
  23 #define CHECK(condition, err_code)                                             \
  24 do {                                                                           \
  25         if (!(condition))                                                      \
  26                 return -(err_code);                                            \
  27 } while (0)
  28
  29 #define CHECK_NAME(name, err_code)                                             \
  30         CHECK((name) &&                                                        \
  31               (name)[0] &&                                                     \
  32               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  33               err_code)
  34
  35 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  36         CHECK((instr) &&                                                       \
  37               (instr)[0] &&                                                    \
  38               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  39                RTE_SWX_INSTRUCTION_SIZE),                                      \
  40               err_code)
  41
  42 #ifndef TRACE_LEVEL
  43 #define TRACE_LEVEL 0
  44 #endif
  45
  46 #if TRACE_LEVEL
  47 #define TRACE(...) printf(__VA_ARGS__)
  48 #else
  49 #define TRACE(...)
  50 #endif
  51
  52 /*
  53  * Environment.
  54  */
  55 #define ntoh64(x) rte_be_to_cpu_64(x)
  56 #define hton64(x) rte_cpu_to_be_64(x)
  57
  58 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  59
  60 #include <rte_malloc.h>
  61
  62 static void *
  63 env_malloc(size_t size, size_t alignment, int numa_node)
  64 {
  65         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  66 }
  67
  68 static void
  69 env_free(void *start, size_t size __rte_unused)
  70 {
  71         rte_free(start);
  72 }
  73
  74 #else
  75
  76 #include <numa.h>
  77
  78 static void *
  79 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  80 {
  81         void *start;
  82
  83         if (numa_available() == -1)
  84                 return NULL;
  85
  86         start = numa_alloc_onnode(size, numa_node);
  87         if (!start)
  88                 return NULL;
  89
  90         memset(start, 0, size);
  91         return start;
  92 }
  93
  94 static void
  95 env_free(void *start, size_t size)
  96 {
  97         if (numa_available() == -1)
  98                 return;
  99
 100         numa_free(start, size);
 101 }
 102
 103 #endif
 104
 105 /*
 106  * Struct.
 107  */
 108 struct field {
 109         char name[RTE_SWX_NAME_SIZE];
 110         uint32_t n_bits;
 111         uint32_t offset;
 112 };
 113
 114 struct struct_type {
 115         TAILQ_ENTRY(struct_type) node;
 116         char name[RTE_SWX_NAME_SIZE];
 117         struct field *fields;
 118         uint32_t n_fields;
 119         uint32_t n_bits;
 120 };
 121
 122 TAILQ_HEAD(struct_type_tailq, struct_type);
 123
 124 /*
 125  * Input port.
 126  */
 127 struct port_in_type {
 128         TAILQ_ENTRY(port_in_type) node;
 129         char name[RTE_SWX_NAME_SIZE];
 130         struct rte_swx_port_in_ops ops;
 131 };
 132
 133 TAILQ_HEAD(port_in_type_tailq, port_in_type);
 134
 135 struct port_in {
 136         TAILQ_ENTRY(port_in) node;
 137         struct port_in_type *type;
 138         void *obj;
 139         uint32_t id;
 140 };
 141
 142 TAILQ_HEAD(port_in_tailq, port_in);
 143
 144 struct port_in_runtime {
 145         rte_swx_port_in_pkt_rx_t pkt_rx;
 146         void *obj;
 147 };
 148
 149 /*
 150  * Output port.
 151  */
 152 struct port_out_type {
 153         TAILQ_ENTRY(port_out_type) node;
 154         char name[RTE_SWX_NAME_SIZE];
 155         struct rte_swx_port_out_ops ops;
 156 };
 157
 158 TAILQ_HEAD(port_out_type_tailq, port_out_type);
 159
 160 struct port_out {
 161         TAILQ_ENTRY(port_out) node;
 162         struct port_out_type *type;
 163         void *obj;
 164         uint32_t id;
 165 };
 166
 167 TAILQ_HEAD(port_out_tailq, port_out);
 168
 169 struct port_out_runtime {
 170         rte_swx_port_out_pkt_tx_t pkt_tx;
 171         rte_swx_port_out_flush_t flush;
 172         void *obj;
 173 };
 174
 175 /*
 176  * Extern object.
 177  */
 178 struct extern_type_member_func {
 179         TAILQ_ENTRY(extern_type_member_func) node;
 180         char name[RTE_SWX_NAME_SIZE];
 181         rte_swx_extern_type_member_func_t func;
 182         uint32_t id;
 183 };
 184
 185 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 186
 187 struct extern_type {
 188         TAILQ_ENTRY(extern_type) node;
 189         char name[RTE_SWX_NAME_SIZE];
 190         struct struct_type *mailbox_struct_type;
 191         rte_swx_extern_type_constructor_t constructor;
 192         rte_swx_extern_type_destructor_t destructor;
 193         struct extern_type_member_func_tailq funcs;
 194         uint32_t n_funcs;
 195 };
 196
 197 TAILQ_HEAD(extern_type_tailq, extern_type);
 198
 199 struct extern_obj {
 200         TAILQ_ENTRY(extern_obj) node;
 201         char name[RTE_SWX_NAME_SIZE];
 202         struct extern_type *type;
 203         void *obj;
 204         uint32_t struct_id;
 205         uint32_t id;
 206 };
 207
 208 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 209
 210 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 211 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 212 #endif
 213
 214 struct extern_obj_runtime {
 215         void *obj;
 216         uint8_t *mailbox;
 217         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 218 };
 219
 220 /*
 221  * Extern function.
 222  */
 223 struct extern_func {
 224         TAILQ_ENTRY(extern_func) node;
 225         char name[RTE_SWX_NAME_SIZE];
 226         struct struct_type *mailbox_struct_type;
 227         rte_swx_extern_func_t func;
 228         uint32_t struct_id;
 229         uint32_t id;
 230 };
 231
 232 TAILQ_HEAD(extern_func_tailq, extern_func);
 233
 234 struct extern_func_runtime {
 235         uint8_t *mailbox;
 236         rte_swx_extern_func_t func;
 237 };
 238
 239 /*
 240  * Header.
 241  */
 242 struct header {
 243         TAILQ_ENTRY(header) node;
 244         char name[RTE_SWX_NAME_SIZE];
 245         struct struct_type *st;
 246         uint32_t struct_id;
 247         uint32_t id;
 248 };
 249
 250 TAILQ_HEAD(header_tailq, header);
 251
 252 struct header_runtime {
 253         uint8_t *ptr0;
 254         uint32_t n_bytes;
 255 };
 256
 257 struct header_out_runtime {
 258         uint8_t *ptr0;
 259         uint8_t *ptr;
 260         uint32_t n_bytes;
 261 };
 262
 263 /*
 264  * Instruction.
 265  */
 266
 267 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 268  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 269  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 270  * when transferred to packet meta-data and in NBO when transferred to packet
 271  * headers.
 272  */
 273
 274 /* Notation conventions:
 275  *    -Header field: H = h.header.field (dst/src)
 276  *    -Meta-data field: M = m.field (dst/src)
 277  *    -Extern object mailbox field: E = e.field (dst/src)
 278  *    -Extern function mailbox field: F = f.field (dst/src)
 279  *    -Table action data field: T = t.field (src only)
 280  *    -Immediate value: I = 32-bit unsigned value (src only)
 281  */
 282
 283 enum instruction_type {
 284         /* rx m.port_in */
 285         INSTR_RX,
 286
 287         /* tx port_out
 288          * port_out = MI
 289          */
 290         INSTR_TX,   /* port_out = M */
 291         INSTR_TX_I, /* port_out = I */
 292
 293         /* extract h.header */
 294         INSTR_HDR_EXTRACT,
 295         INSTR_HDR_EXTRACT2,
 296         INSTR_HDR_EXTRACT3,
 297         INSTR_HDR_EXTRACT4,
 298         INSTR_HDR_EXTRACT5,
 299         INSTR_HDR_EXTRACT6,
 300         INSTR_HDR_EXTRACT7,
 301         INSTR_HDR_EXTRACT8,
 302
 303         /* emit h.header */
 304         INSTR_HDR_EMIT,
 305         INSTR_HDR_EMIT_TX,
 306         INSTR_HDR_EMIT2_TX,
 307         INSTR_HDR_EMIT3_TX,
 308         INSTR_HDR_EMIT4_TX,
 309         INSTR_HDR_EMIT5_TX,
 310         INSTR_HDR_EMIT6_TX,
 311         INSTR_HDR_EMIT7_TX,
 312         INSTR_HDR_EMIT8_TX,
 313
 314         /* validate h.header */
 315         INSTR_HDR_VALIDATE,
 316
 317         /* invalidate h.header */
 318         INSTR_HDR_INVALIDATE,
 319
 320         /* mov dst src
 321          * dst = src
 322          * dst = HMEF, src = HMEFTI
 323          */
 324         INSTR_MOV,    /* dst = MEF, src = MEFT */
 325         INSTR_MOV_MH, /* dst = MEF, src = H */
 326         INSTR_MOV_HM, /* dst = H, src = MEFT */
 327         INSTR_MOV_HH, /* dst = H, src = H */
 328         INSTR_MOV_I,  /* dst = HMEF, src = I */
 329
 330         /* dma h.header t.field
 331          * memcpy(h.header, t.field, sizeof(h.header))
 332          */
 333         INSTR_DMA_HT,
 334         INSTR_DMA_HT2,
 335         INSTR_DMA_HT3,
 336         INSTR_DMA_HT4,
 337         INSTR_DMA_HT5,
 338         INSTR_DMA_HT6,
 339         INSTR_DMA_HT7,
 340         INSTR_DMA_HT8,
 341
 342         /* add dst src
 343          * dst += src
 344          * dst = HMEF, src = HMEFTI
 345          */
 346         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 347         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 348         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 349         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 350         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 351         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 352
 353         /* sub dst src
 354          * dst -= src
 355          * dst = HMEF, src = HMEFTI
 356          */
 357         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 358         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 359         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 360         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 361         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 362         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 363
 364         /* ckadd dst src
 365          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 366          * dst = H, src = {H, h.header}
 367          */
 368         INSTR_ALU_CKADD_FIELD,    /* src = H */
 369         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 370         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 371
 372         /* cksub dst src
 373          * dst = dst '- src
 374          * dst = H, src = H
 375          */
 376         INSTR_ALU_CKSUB_FIELD,
 377
 378         /* and dst src
 379          * dst &= src
 380          * dst = HMEF, src = HMEFTI
 381          */
 382         INSTR_ALU_AND,    /* dst = MEF, src = MEFT */
 383         INSTR_ALU_AND_MH, /* dst = MEF, src = H */
 384         INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
 385         INSTR_ALU_AND_HH, /* dst = H, src = H */
 386         INSTR_ALU_AND_I,  /* dst = HMEF, src = I */
 387
 388         /* or dst src
 389          * dst |= src
 390          * dst = HMEF, src = HMEFTI
 391          */
 392         INSTR_ALU_OR,    /* dst = MEF, src = MEFT */
 393         INSTR_ALU_OR_MH, /* dst = MEF, src = H */
 394         INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
 395         INSTR_ALU_OR_HH, /* dst = H, src = H */
 396         INSTR_ALU_OR_I,  /* dst = HMEF, src = I */
 397
 398         /* xor dst src
 399          * dst ^= src
 400          * dst = HMEF, src = HMEFTI
 401          */
 402         INSTR_ALU_XOR,    /* dst = MEF, src = MEFT */
 403         INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
 404         INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
 405         INSTR_ALU_XOR_HH, /* dst = H, src = H */
 406         INSTR_ALU_XOR_I,  /* dst = HMEF, src = I */
 407
 408         /* shl dst src
 409          * dst <<= src
 410          * dst = HMEF, src = HMEFTI
 411          */
 412         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 413         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 414         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 415         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 416         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 417         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 418
 419         /* shr dst src
 420          * dst >>= src
 421          * dst = HMEF, src = HMEFTI
 422          */
 423         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 424         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 425         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 426         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 427         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 428         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 429
 430         /* regprefetch REGARRAY index
 431          * prefetch REGARRAY[index]
 432          * index = HMEFTI
 433          */
 434         INSTR_REGPREFETCH_RH, /* index = H */
 435         INSTR_REGPREFETCH_RM, /* index = MEFT */
 436         INSTR_REGPREFETCH_RI, /* index = I */
 437
 438         /* regrd dst REGARRAY index
 439          * dst = REGARRAY[index]
 440          * dst = HMEF, index = HMEFTI
 441          */
 442         INSTR_REGRD_HRH, /* dst = H, index = H */
 443         INSTR_REGRD_HRM, /* dst = H, index = MEFT */
 444         INSTR_REGRD_HRI, /* dst = H, index = I */
 445         INSTR_REGRD_MRH, /* dst = MEF, index = H */
 446         INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
 447         INSTR_REGRD_MRI, /* dst = MEF, index = I */
 448
 449         /* regwr REGARRAY index src
 450          * REGARRAY[index] = src
 451          * index = HMEFTI, src = HMEFTI
 452          */
 453         INSTR_REGWR_RHH, /* index = H, src = H */
 454         INSTR_REGWR_RHM, /* index = H, src = MEFT */
 455         INSTR_REGWR_RHI, /* index = H, src = I */
 456         INSTR_REGWR_RMH, /* index = MEFT, src = H */
 457         INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
 458         INSTR_REGWR_RMI, /* index = MEFT, src = I */
 459         INSTR_REGWR_RIH, /* index = I, src = H */
 460         INSTR_REGWR_RIM, /* index = I, src = MEFT */
 461         INSTR_REGWR_RII, /* index = I, src = I */
 462
 463         /* regadd REGARRAY index src
 464          * REGARRAY[index] += src
 465          * index = HMEFTI, src = HMEFTI
 466          */
 467         INSTR_REGADD_RHH, /* index = H, src = H */
 468         INSTR_REGADD_RHM, /* index = H, src = MEFT */
 469         INSTR_REGADD_RHI, /* index = H, src = I */
 470         INSTR_REGADD_RMH, /* index = MEFT, src = H */
 471         INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
 472         INSTR_REGADD_RMI, /* index = MEFT, src = I */
 473         INSTR_REGADD_RIH, /* index = I, src = H */
 474         INSTR_REGADD_RIM, /* index = I, src = MEFT */
 475         INSTR_REGADD_RII, /* index = I, src = I */
 476
 477         /* metprefetch METARRAY index
 478          * prefetch METARRAY[index]
 479          * index = HMEFTI
 480          */
 481         INSTR_METPREFETCH_H, /* index = H */
 482         INSTR_METPREFETCH_M, /* index = MEFT */
 483         INSTR_METPREFETCH_I, /* index = I */
 484
 485         /* meter METARRAY index length color_in color_out
 486          * color_out = meter(METARRAY[index], length, color_in)
 487          * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
 488          */
 489         INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
 490         INSTR_METER_HHI, /* index = H, length = H, color_in = I */
 491         INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
 492         INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
 493         INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
 494         INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
 495         INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
 496         INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
 497         INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
 498         INSTR_METER_IHI, /* index = I, length = H, color_in = I */
 499         INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
 500         INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
 501
 502         /* table TABLE */
 503         INSTR_TABLE,
 504         INSTR_SELECTOR,
 505
 506         /* extern e.obj.func */
 507         INSTR_EXTERN_OBJ,
 508
 509         /* extern f.func */
 510         INSTR_EXTERN_FUNC,
 511
 512         /* jmp LABEL
 513          * Unconditional jump
 514          */
 515         INSTR_JMP,
 516
 517         /* jmpv LABEL h.header
 518          * Jump if header is valid
 519          */
 520         INSTR_JMP_VALID,
 521
 522         /* jmpnv LABEL h.header
 523          * Jump if header is invalid
 524          */
 525         INSTR_JMP_INVALID,
 526
 527         /* jmph LABEL
 528          * Jump if table lookup hit
 529          */
 530         INSTR_JMP_HIT,
 531
 532         /* jmpnh LABEL
 533          * Jump if table lookup miss
 534          */
 535         INSTR_JMP_MISS,
 536
 537         /* jmpa LABEL ACTION
 538          * Jump if action run
 539          */
 540         INSTR_JMP_ACTION_HIT,
 541
 542         /* jmpna LABEL ACTION
 543          * Jump if action not run
 544          */
 545         INSTR_JMP_ACTION_MISS,
 546
 547         /* jmpeq LABEL a b
 548          * Jump if a is equal to b
 549          * a = HMEFT, b = HMEFTI
 550          */
 551         INSTR_JMP_EQ,    /* a = MEFT, b = MEFT */
 552         INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
 553         INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
 554         INSTR_JMP_EQ_HH, /* a = H, b = H */
 555         INSTR_JMP_EQ_I,  /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 556
 557         /* jmpneq LABEL a b
 558          * Jump if a is not equal to b
 559          * a = HMEFT, b = HMEFTI
 560          */
 561         INSTR_JMP_NEQ,    /* a = MEFT, b = MEFT */
 562         INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
 563         INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
 564         INSTR_JMP_NEQ_HH, /* a = H, b = H */
 565         INSTR_JMP_NEQ_I,  /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 566
 567         /* jmplt LABEL a b
 568          * Jump if a is less than b
 569          * a = HMEFT, b = HMEFTI
 570          */
 571         INSTR_JMP_LT,    /* a = MEFT, b = MEFT */
 572         INSTR_JMP_LT_MH, /* a = MEFT, b = H */
 573         INSTR_JMP_LT_HM, /* a = H, b = MEFT */
 574         INSTR_JMP_LT_HH, /* a = H, b = H */
 575         INSTR_JMP_LT_MI, /* a = MEFT, b = I */
 576         INSTR_JMP_LT_HI, /* a = H, b = I */
 577
 578         /* jmpgt LABEL a b
 579          * Jump if a is greater than b
 580          * a = HMEFT, b = HMEFTI
 581          */
 582         INSTR_JMP_GT,    /* a = MEFT, b = MEFT */
 583         INSTR_JMP_GT_MH, /* a = MEFT, b = H */
 584         INSTR_JMP_GT_HM, /* a = H, b = MEFT */
 585         INSTR_JMP_GT_HH, /* a = H, b = H */
 586         INSTR_JMP_GT_MI, /* a = MEFT, b = I */
 587         INSTR_JMP_GT_HI, /* a = H, b = I */
 588
 589         /* return
 590          * Return from action
 591          */
 592         INSTR_RETURN,
 593 };
 594
 595 struct instr_operand {
 596         uint8_t struct_id;
 597         uint8_t n_bits;
 598         uint8_t offset;
 599         uint8_t pad;
 600 };
 601
 602 struct instr_io {
 603         struct {
 604                 union {
 605                         struct {
 606                                 uint8_t offset;
 607                                 uint8_t n_bits;
 608                                 uint8_t pad[2];
 609                         };
 610
 611                         uint32_t val;
 612                 };
 613         } io;
 614
 615         struct {
 616                 uint8_t header_id[8];
 617                 uint8_t struct_id[8];
 618                 uint8_t n_bytes[8];
 619         } hdr;
 620 };
 621
 622 struct instr_hdr_validity {
 623         uint8_t header_id;
 624 };
 625
 626 struct instr_table {
 627         uint8_t table_id;
 628 };
 629
 630 struct instr_extern_obj {
 631         uint8_t ext_obj_id;
 632         uint8_t func_id;
 633 };
 634
 635 struct instr_extern_func {
 636         uint8_t ext_func_id;
 637 };
 638
 639 struct instr_dst_src {
 640         struct instr_operand dst;
 641         union {
 642                 struct instr_operand src;
 643                 uint64_t src_val;
 644         };
 645 };
 646
 647 struct instr_regarray {
 648         uint8_t regarray_id;
 649         uint8_t pad[3];
 650
 651         union {
 652                 struct instr_operand idx;
 653                 uint32_t idx_val;
 654         };
 655
 656         union {
 657                 struct instr_operand dstsrc;
 658                 uint64_t dstsrc_val;
 659         };
 660 };
 661
 662 struct instr_meter {
 663         uint8_t metarray_id;
 664         uint8_t pad[3];
 665
 666         union {
 667                 struct instr_operand idx;
 668                 uint32_t idx_val;
 669         };
 670
 671         struct instr_operand length;
 672
 673         union {
 674                 struct instr_operand color_in;
 675                 uint32_t color_in_val;
 676         };
 677
 678         struct instr_operand color_out;
 679 };
 680
 681 struct instr_dma {
 682         struct {
 683                 uint8_t header_id[8];
 684                 uint8_t struct_id[8];
 685         } dst;
 686
 687         struct {
 688                 uint8_t offset[8];
 689         } src;
 690
 691         uint16_t n_bytes[8];
 692 };
 693
 694 struct instr_jmp {
 695         struct instruction *ip;
 696
 697         union {
 698                 struct instr_operand a;
 699                 uint8_t header_id;
 700                 uint8_t action_id;
 701         };
 702
 703         union {
 704                 struct instr_operand b;
 705                 uint64_t b_val;
 706         };
 707 };
 708
 709 struct instruction {
 710         enum instruction_type type;
 711         union {
 712                 struct instr_io io;
 713                 struct instr_hdr_validity valid;
 714                 struct instr_dst_src mov;
 715                 struct instr_regarray regarray;
 716                 struct instr_meter meter;
 717                 struct instr_dma dma;
 718                 struct instr_dst_src alu;
 719                 struct instr_table table;
 720                 struct instr_extern_obj ext_obj;
 721                 struct instr_extern_func ext_func;
 722                 struct instr_jmp jmp;
 723         };
 724 };
 725
 726 struct instruction_data {
 727         char label[RTE_SWX_NAME_SIZE];
 728         char jmp_label[RTE_SWX_NAME_SIZE];
 729         uint32_t n_users; /* user = jmp instruction to this instruction. */
 730         int invalid;
 731 };
 732
 733 /*
 734  * Action.
 735  */
 736 struct action {
 737         TAILQ_ENTRY(action) node;
 738         char name[RTE_SWX_NAME_SIZE];
 739         struct struct_type *st;
 740         int *args_endianness; /* 0 = Host Byte Order (HBO). */
 741         struct instruction *instructions;
 742         uint32_t n_instructions;
 743         uint32_t id;
 744 };
 745
 746 TAILQ_HEAD(action_tailq, action);
 747
 748 /*
 749  * Table.
 750  */
 751 struct table_type {
 752         TAILQ_ENTRY(table_type) node;
 753         char name[RTE_SWX_NAME_SIZE];
 754         enum rte_swx_table_match_type match_type;
 755         struct rte_swx_table_ops ops;
 756 };
 757
 758 TAILQ_HEAD(table_type_tailq, table_type);
 759
 760 struct match_field {
 761         enum rte_swx_table_match_type match_type;
 762         struct field *field;
 763 };
 764
 765 struct table {
 766         TAILQ_ENTRY(table) node;
 767         char name[RTE_SWX_NAME_SIZE];
 768         char args[RTE_SWX_NAME_SIZE];
 769         struct table_type *type; /* NULL when n_fields == 0. */
 770
 771         /* Match. */
 772         struct match_field *fields;
 773         uint32_t n_fields;
 774         struct header *header; /* Only valid when n_fields > 0. */
 775
 776         /* Action. */
 777         struct action **actions;
 778         struct action *default_action;
 779         uint8_t *default_action_data;
 780         uint32_t n_actions;
 781         int default_action_is_const;
 782         uint32_t action_data_size_max;
 783
 784         uint32_t size;
 785         uint32_t id;
 786 };
 787
 788 TAILQ_HEAD(table_tailq, table);
 789
 790 struct table_runtime {
 791         rte_swx_table_lookup_t func;
 792         void *mailbox;
 793         uint8_t **key;
 794 };
 795
 796 struct table_statistics {
 797         uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
 798         uint64_t *n_pkts_action;
 799 };
 800
 801 /*
 802  * Selector.
 803  */
 804 struct selector {
 805         TAILQ_ENTRY(selector) node;
 806         char name[RTE_SWX_NAME_SIZE];
 807
 808         struct field *group_id_field;
 809         struct field **selector_fields;
 810         uint32_t n_selector_fields;
 811         struct header *selector_header;
 812         struct field *member_id_field;
 813
 814         uint32_t n_groups_max;
 815         uint32_t n_members_per_group_max;
 816
 817         uint32_t id;
 818 };
 819
 820 TAILQ_HEAD(selector_tailq, selector);
 821
 822 struct selector_runtime {
 823         void *mailbox;
 824         uint8_t **group_id_buffer;
 825         uint8_t **selector_buffer;
 826         uint8_t **member_id_buffer;
 827 };
 828
 829 struct selector_statistics {
 830         uint64_t n_pkts;
 831 };
 832
 833 /*
 834  * Register array.
 835  */
 836 struct regarray {
 837         TAILQ_ENTRY(regarray) node;
 838         char name[RTE_SWX_NAME_SIZE];
 839         uint64_t init_val;
 840         uint32_t size;
 841         uint32_t id;
 842 };
 843
 844 TAILQ_HEAD(regarray_tailq, regarray);
 845
 846 struct regarray_runtime {
 847         uint64_t *regarray;
 848         uint32_t size_mask;
 849 };
 850
 851 /*
 852  * Meter array.
 853  */
 854 struct meter_profile {
 855         TAILQ_ENTRY(meter_profile) node;
 856         char name[RTE_SWX_NAME_SIZE];
 857         struct rte_meter_trtcm_params params;
 858         struct rte_meter_trtcm_profile profile;
 859         uint32_t n_users;
 860 };
 861
 862 TAILQ_HEAD(meter_profile_tailq, meter_profile);
 863
 864 struct metarray {
 865         TAILQ_ENTRY(metarray) node;
 866         char name[RTE_SWX_NAME_SIZE];
 867         uint32_t size;
 868         uint32_t id;
 869 };
 870
 871 TAILQ_HEAD(metarray_tailq, metarray);
 872
 873 struct meter {
 874         struct rte_meter_trtcm m;
 875         struct meter_profile *profile;
 876         enum rte_color color_mask;
 877         uint8_t pad[20];
 878
 879         uint64_t n_pkts[RTE_COLORS];
 880         uint64_t n_bytes[RTE_COLORS];
 881 };
 882
 883 struct metarray_runtime {
 884         struct meter *metarray;
 885         uint32_t size_mask;
 886 };
 887
 888 /*
 889  * Pipeline.
 890  */
 891 struct thread {
 892         /* Packet. */
 893         struct rte_swx_pkt pkt;
 894         uint8_t *ptr;
 895
 896         /* Structures. */
 897         uint8_t **structs;
 898
 899         /* Packet headers. */
 900         struct header_runtime *headers; /* Extracted or generated headers. */
 901         struct header_out_runtime *headers_out; /* Emitted headers. */
 902         uint8_t *header_storage;
 903         uint8_t *header_out_storage;
 904         uint64_t valid_headers;
 905         uint32_t n_headers_out;
 906
 907         /* Packet meta-data. */
 908         uint8_t *metadata;
 909
 910         /* Tables. */
 911         struct table_runtime *tables;
 912         struct selector_runtime *selectors;
 913         struct rte_swx_table_state *table_state;
 914         uint64_t action_id;
 915         int hit; /* 0 = Miss, 1 = Hit. */
 916
 917         /* Extern objects and functions. */
 918         struct extern_obj_runtime *extern_objs;
 919         struct extern_func_runtime *extern_funcs;
 920
 921         /* Instructions. */
 922         struct instruction *ip;
 923         struct instruction *ret;
 924 };
 925
 926 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 927 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 928 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 929
 930 #define HEADER_VALID(thread, header_id) \
 931         MASK64_BIT_GET((thread)->valid_headers, header_id)
 932
 933 #define ALU(thread, ip, operator)  \
 934 {                                                                              \
 935         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 936         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 937         uint64_t dst64 = *dst64_ptr;                                           \
 938         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 939         uint64_t dst = dst64 & dst64_mask;                                     \
 940                                                                                \
 941         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 942         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 943         uint64_t src64 = *src64_ptr;                                           \
 944         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 945         uint64_t src = src64 & src64_mask;                                     \
 946                                                                                \
 947         uint64_t result = dst operator src;                                    \
 948                                                                                \
 949         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 950 }
 951
 952 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 953
 954 #define ALU_MH(thread, ip, operator)  \
 955 {                                                                              \
 956         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 957         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 958         uint64_t dst64 = *dst64_ptr;                                           \
 959         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 960         uint64_t dst = dst64 & dst64_mask;                                     \
 961                                                                                \
 962         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 963         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 964         uint64_t src64 = *src64_ptr;                                           \
 965         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 966                                                                                \
 967         uint64_t result = dst operator src;                                    \
 968                                                                                \
 969         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 970 }
 971
 972 #define ALU_HM(thread, ip, operator)  \
 973 {                                                                              \
 974         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 975         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 976         uint64_t dst64 = *dst64_ptr;                                           \
 977         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 978         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 979                                                                                \
 980         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 981         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 982         uint64_t src64 = *src64_ptr;                                           \
 983         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 984         uint64_t src = src64 & src64_mask;                                     \
 985                                                                                \
 986         uint64_t result = dst operator src;                                    \
 987         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 988                                                                                \
 989         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 990 }
 991
 992 #define ALU_HM_FAST(thread, ip, operator)  \
 993 {                                                                                 \
 994         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];         \
 995         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];      \
 996         uint64_t dst64 = *dst64_ptr;                                              \
 997         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);          \
 998         uint64_t dst = dst64 & dst64_mask;                                        \
 999                                                                                   \
1000         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];         \
1001         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];      \
1002         uint64_t src64 = *src64_ptr;                                              \
1003         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);          \
1004         uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1005                                                                                   \
1006         uint64_t result = dst operator src;                                       \
1007                                                                                   \
1008         *dst64_ptr = (dst64 & ~dst64_mask) | result;                              \
1009 }
1010
1011 #define ALU_HH(thread, ip, operator)  \
1012 {                                                                              \
1013         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1014         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1015         uint64_t dst64 = *dst64_ptr;                                           \
1016         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1017         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
1018                                                                                \
1019         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
1020         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
1021         uint64_t src64 = *src64_ptr;                                           \
1022         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
1023                                                                                \
1024         uint64_t result = dst operator src;                                    \
1025         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1026                                                                                \
1027         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1028 }
1029
1030 #define ALU_HH_FAST(thread, ip, operator)  \
1031 {                                                                                             \
1032         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];                     \
1033         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];                  \
1034         uint64_t dst64 = *dst64_ptr;                                                          \
1035         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);                      \
1036         uint64_t dst = dst64 & dst64_mask;                                                    \
1037                                                                                               \
1038         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];                     \
1039         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];                  \
1040         uint64_t src64 = *src64_ptr;                                                          \
1041         uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1042                                                                                               \
1043         uint64_t result = dst operator src;                                                   \
1044                                                                                               \
1045         *dst64_ptr = (dst64 & ~dst64_mask) | result;                                          \
1046 }
1047
1048 #else
1049
1050 #define ALU_MH ALU
1051 #define ALU_HM ALU
1052 #define ALU_HM_FAST ALU
1053 #define ALU_HH ALU
1054 #define ALU_HH_FAST ALU
1055
1056 #endif
1057
1058 #define ALU_I(thread, ip, operator)  \
1059 {                                                                              \
1060         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1061         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1062         uint64_t dst64 = *dst64_ptr;                                           \
1063         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1064         uint64_t dst = dst64 & dst64_mask;                                     \
1065                                                                                \
1066         uint64_t src = (ip)->alu.src_val;                                      \
1067                                                                                \
1068         uint64_t result = dst operator src;                                    \
1069                                                                                \
1070         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
1071 }
1072
1073 #define ALU_MI ALU_I
1074
1075 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1076
1077 #define ALU_HI(thread, ip, operator)  \
1078 {                                                                              \
1079         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1080         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1081         uint64_t dst64 = *dst64_ptr;                                           \
1082         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1083         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
1084                                                                                \
1085         uint64_t src = (ip)->alu.src_val;                                      \
1086                                                                                \
1087         uint64_t result = dst operator src;                                    \
1088         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1089                                                                                \
1090         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1091 }
1092
1093 #else
1094
1095 #define ALU_HI ALU_I
1096
1097 #endif
1098
1099 #define MOV(thread, ip)  \
1100 {                                                                              \
1101         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1102         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1103         uint64_t dst64 = *dst64_ptr;                                           \
1104         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1105                                                                                \
1106         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1107         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1108         uint64_t src64 = *src64_ptr;                                           \
1109         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1110         uint64_t src = src64 & src64_mask;                                     \
1111                                                                                \
1112         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1113 }
1114
1115 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1116
1117 #define MOV_MH(thread, ip)  \
1118 {                                                                              \
1119         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1120         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1121         uint64_t dst64 = *dst64_ptr;                                           \
1122         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1123                                                                                \
1124         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1125         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1126         uint64_t src64 = *src64_ptr;                                           \
1127         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
1128                                                                                \
1129         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1130 }
1131
1132 #define MOV_HM(thread, ip)  \
1133 {                                                                              \
1134         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1135         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1136         uint64_t dst64 = *dst64_ptr;                                           \
1137         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1138                                                                                \
1139         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1140         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1141         uint64_t src64 = *src64_ptr;                                           \
1142         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1143         uint64_t src = src64 & src64_mask;                                     \
1144                                                                                \
1145         src = hton64(src) >> (64 - (ip)->mov.dst.n_bits);                      \
1146         *dst64_ptr = (dst64 & ~dst64_mask) | src;                              \
1147 }
1148
1149 #define MOV_HH(thread, ip)  \
1150 {                                                                              \
1151         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1152         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1153         uint64_t dst64 = *dst64_ptr;                                           \
1154         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1155                                                                                \
1156         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1157         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1158         uint64_t src64 = *src64_ptr;                                           \
1159                                                                                \
1160         uint64_t src = src64 << (64 - (ip)->mov.src.n_bits);                   \
1161         src = src >> (64 - (ip)->mov.dst.n_bits);                              \
1162         *dst64_ptr = (dst64 & ~dst64_mask) | src;                              \
1163 }
1164
1165 #else
1166
1167 #define MOV_MH MOV
1168 #define MOV_HM MOV
1169 #define MOV_HH MOV
1170
1171 #endif
1172
1173 #define MOV_I(thread, ip)  \
1174 {                                                                              \
1175         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1176         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1177         uint64_t dst64 = *dst64_ptr;                                           \
1178         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1179                                                                                \
1180         uint64_t src = (ip)->mov.src_val;                                      \
1181                                                                                \
1182         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1183 }
1184
1185 #define JMP_CMP(thread, ip, operator)  \
1186 {                                                                              \
1187         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1188         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1189         uint64_t a64 = *a64_ptr;                                               \
1190         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1191         uint64_t a = a64 & a64_mask;                                           \
1192                                                                                \
1193         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1194         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1195         uint64_t b64 = *b64_ptr;                                               \
1196         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1197         uint64_t b = b64 & b64_mask;                                           \
1198                                                                                \
1199         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1200 }
1201
1202 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1203
1204 #define JMP_CMP_MH(thread, ip, operator)  \
1205 {                                                                              \
1206         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1207         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1208         uint64_t a64 = *a64_ptr;                                               \
1209         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1210         uint64_t a = a64 & a64_mask;                                           \
1211                                                                                \
1212         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1213         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1214         uint64_t b64 = *b64_ptr;                                               \
1215         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1216                                                                                \
1217         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1218 }
1219
1220 #define JMP_CMP_HM(thread, ip, operator)  \
1221 {                                                                              \
1222         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1223         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1224         uint64_t a64 = *a64_ptr;                                               \
1225         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1226                                                                                \
1227         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1228         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1229         uint64_t b64 = *b64_ptr;                                               \
1230         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1231         uint64_t b = b64 & b64_mask;                                           \
1232                                                                                \
1233         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1234 }
1235
1236 #define JMP_CMP_HH(thread, ip, operator)  \
1237 {                                                                              \
1238         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1239         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1240         uint64_t a64 = *a64_ptr;                                               \
1241         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1242                                                                                \
1243         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1244         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1245         uint64_t b64 = *b64_ptr;                                               \
1246         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1247                                                                                \
1248         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1249 }
1250
1251 #define JMP_CMP_HH_FAST(thread, ip, operator)  \
1252 {                                                                              \
1253         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1254         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1255         uint64_t a64 = *a64_ptr;                                               \
1256         uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits);                         \
1257                                                                                \
1258         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1259         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1260         uint64_t b64 = *b64_ptr;                                               \
1261         uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits);                         \
1262                                                                                \
1263         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1264 }
1265
1266 #else
1267
1268 #define JMP_CMP_MH JMP_CMP
1269 #define JMP_CMP_HM JMP_CMP
1270 #define JMP_CMP_HH JMP_CMP
1271 #define JMP_CMP_HH_FAST JMP_CMP
1272
1273 #endif
1274
1275 #define JMP_CMP_I(thread, ip, operator)  \
1276 {                                                                              \
1277         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1278         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1279         uint64_t a64 = *a64_ptr;                                               \
1280         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1281         uint64_t a = a64 & a64_mask;                                           \
1282                                                                                \
1283         uint64_t b = (ip)->jmp.b_val;                                          \
1284                                                                                \
1285         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1286 }
1287
1288 #define JMP_CMP_MI JMP_CMP_I
1289
1290 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1291
1292 #define JMP_CMP_HI(thread, ip, operator)  \
1293 {                                                                              \
1294         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1295         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1296         uint64_t a64 = *a64_ptr;                                               \
1297         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1298                                                                                \
1299         uint64_t b = (ip)->jmp.b_val;                                          \
1300                                                                                \
1301         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1302 }
1303
1304 #else
1305
1306 #define JMP_CMP_HI JMP_CMP_I
1307
1308 #endif
1309
1310 #define METADATA_READ(thread, offset, n_bits)                                  \
1311 ({                                                                             \
1312         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1313         uint64_t m64 = *m64_ptr;                                               \
1314         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1315         (m64 & m64_mask);                                                      \
1316 })
1317
1318 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
1319 {                                                                              \
1320         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1321         uint64_t m64 = *m64_ptr;                                               \
1322         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1323                                                                                \
1324         uint64_t m_new = value;                                                \
1325                                                                                \
1326         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
1327 }
1328
1329 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1330 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1331 #endif
1332
1333 struct rte_swx_pipeline {
1334         struct struct_type_tailq struct_types;
1335         struct port_in_type_tailq port_in_types;
1336         struct port_in_tailq ports_in;
1337         struct port_out_type_tailq port_out_types;
1338         struct port_out_tailq ports_out;
1339         struct extern_type_tailq extern_types;
1340         struct extern_obj_tailq extern_objs;
1341         struct extern_func_tailq extern_funcs;
1342         struct header_tailq headers;
1343         struct struct_type *metadata_st;
1344         uint32_t metadata_struct_id;
1345         struct action_tailq actions;
1346         struct table_type_tailq table_types;
1347         struct table_tailq tables;
1348         struct selector_tailq selectors;
1349         struct regarray_tailq regarrays;
1350         struct meter_profile_tailq meter_profiles;
1351         struct metarray_tailq metarrays;
1352
1353         struct port_in_runtime *in;
1354         struct port_out_runtime *out;
1355         struct instruction **action_instructions;
1356         struct rte_swx_table_state *table_state;
1357         struct table_statistics *table_stats;
1358         struct selector_statistics *selector_stats;
1359         struct regarray_runtime *regarray_runtime;
1360         struct metarray_runtime *metarray_runtime;
1361         struct instruction *instructions;
1362         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1363
1364         uint32_t n_structs;
1365         uint32_t n_ports_in;
1366         uint32_t n_ports_out;
1367         uint32_t n_extern_objs;
1368         uint32_t n_extern_funcs;
1369         uint32_t n_actions;
1370         uint32_t n_tables;
1371         uint32_t n_selectors;
1372         uint32_t n_regarrays;
1373         uint32_t n_metarrays;
1374         uint32_t n_headers;
1375         uint32_t thread_id;
1376         uint32_t port_id;
1377         uint32_t n_instructions;
1378         int build_done;
1379         int numa_node;
1380 };
1381
1382 /*
1383  * Struct.
1384  */
1385 static struct struct_type *
1386 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1387 {
1388         struct struct_type *elem;
1389
1390         TAILQ_FOREACH(elem, &p->struct_types, node)
1391                 if (strcmp(elem->name, name) == 0)
1392                         return elem;
1393
1394         return NULL;
1395 }
1396
1397 static struct field *
1398 struct_type_field_find(struct struct_type *st, const char *name)
1399 {
1400         uint32_t i;
1401
1402         for (i = 0; i < st->n_fields; i++) {
1403                 struct field *f = &st->fields[i];
1404
1405                 if (strcmp(f->name, name) == 0)
1406                         return f;
1407         }
1408
1409         return NULL;
1410 }
1411
1412 int
1413 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1414                                       const char *name,
1415                                       struct rte_swx_field_params *fields,
1416                                       uint32_t n_fields)
1417 {
1418         struct struct_type *st;
1419         uint32_t i;
1420
1421         CHECK(p, EINVAL);
1422         CHECK_NAME(name, EINVAL);
1423         CHECK(fields, EINVAL);
1424         CHECK(n_fields, EINVAL);
1425
1426         for (i = 0; i < n_fields; i++) {
1427                 struct rte_swx_field_params *f = &fields[i];
1428                 uint32_t j;
1429
1430                 CHECK_NAME(f->name, EINVAL);
1431                 CHECK(f->n_bits, EINVAL);
1432                 CHECK(f->n_bits <= 64, EINVAL);
1433                 CHECK((f->n_bits & 7) == 0, EINVAL);
1434
1435                 for (j = 0; j < i; j++) {
1436                         struct rte_swx_field_params *f_prev = &fields[j];
1437
1438                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
1439                 }
1440         }
1441
1442         CHECK(!struct_type_find(p, name), EEXIST);
1443
1444         /* Node allocation. */
1445         st = calloc(1, sizeof(struct struct_type));
1446         CHECK(st, ENOMEM);
1447
1448         st->fields = calloc(n_fields, sizeof(struct field));
1449         if (!st->fields) {
1450                 free(st);
1451                 CHECK(0, ENOMEM);
1452         }
1453
1454         /* Node initialization. */
1455         strcpy(st->name, name);
1456         for (i = 0; i < n_fields; i++) {
1457                 struct field *dst = &st->fields[i];
1458                 struct rte_swx_field_params *src = &fields[i];
1459
1460                 strcpy(dst->name, src->name);
1461                 dst->n_bits = src->n_bits;
1462                 dst->offset = st->n_bits;
1463
1464                 st->n_bits += src->n_bits;
1465         }
1466         st->n_fields = n_fields;
1467
1468         /* Node add to tailq. */
1469         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1470
1471         return 0;
1472 }
1473
1474 static int
1475 struct_build(struct rte_swx_pipeline *p)
1476 {
1477         uint32_t i;
1478
1479         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1480                 struct thread *t = &p->threads[i];
1481
1482                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1483                 CHECK(t->structs, ENOMEM);
1484         }
1485
1486         return 0;
1487 }
1488
1489 static void
1490 struct_build_free(struct rte_swx_pipeline *p)
1491 {
1492         uint32_t i;
1493
1494         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1495                 struct thread *t = &p->threads[i];
1496
1497                 free(t->structs);
1498                 t->structs = NULL;
1499         }
1500 }
1501
1502 static void
1503 struct_free(struct rte_swx_pipeline *p)
1504 {
1505         struct_build_free(p);
1506
1507         /* Struct types. */
1508         for ( ; ; ) {
1509                 struct struct_type *elem;
1510
1511                 elem = TAILQ_FIRST(&p->struct_types);
1512                 if (!elem)
1513                         break;
1514
1515                 TAILQ_REMOVE(&p->struct_types, elem, node);
1516                 free(elem->fields);
1517                 free(elem);
1518         }
1519 }
1520
1521 /*
1522  * Input port.
1523  */
1524 static struct port_in_type *
1525 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1526 {
1527         struct port_in_type *elem;
1528
1529         if (!name)
1530                 return NULL;
1531
1532         TAILQ_FOREACH(elem, &p->port_in_types, node)
1533                 if (strcmp(elem->name, name) == 0)
1534                         return elem;
1535
1536         return NULL;
1537 }
1538
1539 int
1540 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1541                                        const char *name,
1542                                        struct rte_swx_port_in_ops *ops)
1543 {
1544         struct port_in_type *elem;
1545
1546         CHECK(p, EINVAL);
1547         CHECK_NAME(name, EINVAL);
1548         CHECK(ops, EINVAL);
1549         CHECK(ops->create, EINVAL);
1550         CHECK(ops->free, EINVAL);
1551         CHECK(ops->pkt_rx, EINVAL);
1552         CHECK(ops->stats_read, EINVAL);
1553
1554         CHECK(!port_in_type_find(p, name), EEXIST);
1555
1556         /* Node allocation. */
1557         elem = calloc(1, sizeof(struct port_in_type));
1558         CHECK(elem, ENOMEM);
1559
1560         /* Node initialization. */
1561         strcpy(elem->name, name);
1562         memcpy(&elem->ops, ops, sizeof(*ops));
1563
1564         /* Node add to tailq. */
1565         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1566
1567         return 0;
1568 }
1569
1570 static struct port_in *
1571 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1572 {
1573         struct port_in *port;
1574
1575         TAILQ_FOREACH(port, &p->ports_in, node)
1576                 if (port->id == port_id)
1577                         return port;
1578
1579         return NULL;
1580 }
1581
1582 int
1583 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1584                                 uint32_t port_id,
1585                                 const char *port_type_name,
1586                                 void *args)
1587 {
1588         struct port_in_type *type = NULL;
1589         struct port_in *port = NULL;
1590         void *obj = NULL;
1591
1592         CHECK(p, EINVAL);
1593
1594         CHECK(!port_in_find(p, port_id), EINVAL);
1595
1596         CHECK_NAME(port_type_name, EINVAL);
1597         type = port_in_type_find(p, port_type_name);
1598         CHECK(type, EINVAL);
1599
1600         obj = type->ops.create(args);
1601         CHECK(obj, ENODEV);
1602
1603         /* Node allocation. */
1604         port = calloc(1, sizeof(struct port_in));
1605         CHECK(port, ENOMEM);
1606
1607         /* Node initialization. */
1608         port->type = type;
1609         port->obj = obj;
1610         port->id = port_id;
1611
1612         /* Node add to tailq. */
1613         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1614         if (p->n_ports_in < port_id + 1)
1615                 p->n_ports_in = port_id + 1;
1616
1617         return 0;
1618 }
1619
1620 static int
1621 port_in_build(struct rte_swx_pipeline *p)
1622 {
1623         struct port_in *port;
1624         uint32_t i;
1625
1626         CHECK(p->n_ports_in, EINVAL);
1627         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1628
1629         for (i = 0; i < p->n_ports_in; i++)
1630                 CHECK(port_in_find(p, i), EINVAL);
1631
1632         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1633         CHECK(p->in, ENOMEM);
1634
1635         TAILQ_FOREACH(port, &p->ports_in, node) {
1636                 struct port_in_runtime *in = &p->in[port->id];
1637
1638                 in->pkt_rx = port->type->ops.pkt_rx;
1639                 in->obj = port->obj;
1640         }
1641
1642         return 0;
1643 }
1644
1645 static void
1646 port_in_build_free(struct rte_swx_pipeline *p)
1647 {
1648         free(p->in);
1649         p->in = NULL;
1650 }
1651
1652 static void
1653 port_in_free(struct rte_swx_pipeline *p)
1654 {
1655         port_in_build_free(p);
1656
1657         /* Input ports. */
1658         for ( ; ; ) {
1659                 struct port_in *port;
1660
1661                 port = TAILQ_FIRST(&p->ports_in);
1662                 if (!port)
1663                         break;
1664
1665                 TAILQ_REMOVE(&p->ports_in, port, node);
1666                 port->type->ops.free(port->obj);
1667                 free(port);
1668         }
1669
1670         /* Input port types. */
1671         for ( ; ; ) {
1672                 struct port_in_type *elem;
1673
1674                 elem = TAILQ_FIRST(&p->port_in_types);
1675                 if (!elem)
1676                         break;
1677
1678                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1679                 free(elem);
1680         }
1681 }
1682
1683 /*
1684  * Output port.
1685  */
1686 static struct port_out_type *
1687 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1688 {
1689         struct port_out_type *elem;
1690
1691         if (!name)
1692                 return NULL;
1693
1694         TAILQ_FOREACH(elem, &p->port_out_types, node)
1695                 if (!strcmp(elem->name, name))
1696                         return elem;
1697
1698         return NULL;
1699 }
1700
1701 int
1702 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1703                                         const char *name,
1704                                         struct rte_swx_port_out_ops *ops)
1705 {
1706         struct port_out_type *elem;
1707
1708         CHECK(p, EINVAL);
1709         CHECK_NAME(name, EINVAL);
1710         CHECK(ops, EINVAL);
1711         CHECK(ops->create, EINVAL);
1712         CHECK(ops->free, EINVAL);
1713         CHECK(ops->pkt_tx, EINVAL);
1714         CHECK(ops->stats_read, EINVAL);
1715
1716         CHECK(!port_out_type_find(p, name), EEXIST);
1717
1718         /* Node allocation. */
1719         elem = calloc(1, sizeof(struct port_out_type));
1720         CHECK(elem, ENOMEM);
1721
1722         /* Node initialization. */
1723         strcpy(elem->name, name);
1724         memcpy(&elem->ops, ops, sizeof(*ops));
1725
1726         /* Node add to tailq. */
1727         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1728
1729         return 0;
1730 }
1731
1732 static struct port_out *
1733 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1734 {
1735         struct port_out *port;
1736
1737         TAILQ_FOREACH(port, &p->ports_out, node)
1738                 if (port->id == port_id)
1739                         return port;
1740
1741         return NULL;
1742 }
1743
1744 int
1745 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1746                                  uint32_t port_id,
1747                                  const char *port_type_name,
1748                                  void *args)
1749 {
1750         struct port_out_type *type = NULL;
1751         struct port_out *port = NULL;
1752         void *obj = NULL;
1753
1754         CHECK(p, EINVAL);
1755
1756         CHECK(!port_out_find(p, port_id), EINVAL);
1757
1758         CHECK_NAME(port_type_name, EINVAL);
1759         type = port_out_type_find(p, port_type_name);
1760         CHECK(type, EINVAL);
1761
1762         obj = type->ops.create(args);
1763         CHECK(obj, ENODEV);
1764
1765         /* Node allocation. */
1766         port = calloc(1, sizeof(struct port_out));
1767         CHECK(port, ENOMEM);
1768
1769         /* Node initialization. */
1770         port->type = type;
1771         port->obj = obj;
1772         port->id = port_id;
1773
1774         /* Node add to tailq. */
1775         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1776         if (p->n_ports_out < port_id + 1)
1777                 p->n_ports_out = port_id + 1;
1778
1779         return 0;
1780 }
1781
1782 static int
1783 port_out_build(struct rte_swx_pipeline *p)
1784 {
1785         struct port_out *port;
1786         uint32_t i;
1787
1788         CHECK(p->n_ports_out, EINVAL);
1789
1790         for (i = 0; i < p->n_ports_out; i++)
1791                 CHECK(port_out_find(p, i), EINVAL);
1792
1793         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1794         CHECK(p->out, ENOMEM);
1795
1796         TAILQ_FOREACH(port, &p->ports_out, node) {
1797                 struct port_out_runtime *out = &p->out[port->id];
1798
1799                 out->pkt_tx = port->type->ops.pkt_tx;
1800                 out->flush = port->type->ops.flush;
1801                 out->obj = port->obj;
1802         }
1803
1804         return 0;
1805 }
1806
1807 static void
1808 port_out_build_free(struct rte_swx_pipeline *p)
1809 {
1810         free(p->out);
1811         p->out = NULL;
1812 }
1813
1814 static void
1815 port_out_free(struct rte_swx_pipeline *p)
1816 {
1817         port_out_build_free(p);
1818
1819         /* Output ports. */
1820         for ( ; ; ) {
1821                 struct port_out *port;
1822
1823                 port = TAILQ_FIRST(&p->ports_out);
1824                 if (!port)
1825                         break;
1826
1827                 TAILQ_REMOVE(&p->ports_out, port, node);
1828                 port->type->ops.free(port->obj);
1829                 free(port);
1830         }
1831
1832         /* Output port types. */
1833         for ( ; ; ) {
1834                 struct port_out_type *elem;
1835
1836                 elem = TAILQ_FIRST(&p->port_out_types);
1837                 if (!elem)
1838                         break;
1839
1840                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1841                 free(elem);
1842         }
1843 }
1844
1845 /*
1846  * Extern object.
1847  */
1848 static struct extern_type *
1849 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1850 {
1851         struct extern_type *elem;
1852
1853         TAILQ_FOREACH(elem, &p->extern_types, node)
1854                 if (strcmp(elem->name, name) == 0)
1855                         return elem;
1856
1857         return NULL;
1858 }
1859
1860 static struct extern_type_member_func *
1861 extern_type_member_func_find(struct extern_type *type, const char *name)
1862 {
1863         struct extern_type_member_func *elem;
1864
1865         TAILQ_FOREACH(elem, &type->funcs, node)
1866                 if (strcmp(elem->name, name) == 0)
1867                         return elem;
1868
1869         return NULL;
1870 }
1871
1872 static struct extern_obj *
1873 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1874 {
1875         struct extern_obj *elem;
1876
1877         TAILQ_FOREACH(elem, &p->extern_objs, node)
1878                 if (strcmp(elem->name, name) == 0)
1879                         return elem;
1880
1881         return NULL;
1882 }
1883
1884 static struct extern_type_member_func *
1885 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1886                              const char *name,
1887                              struct extern_obj **obj)
1888 {
1889         struct extern_obj *object;
1890         struct extern_type_member_func *func;
1891         char *object_name, *func_name;
1892
1893         if (name[0] != 'e' || name[1] != '.')
1894                 return NULL;
1895
1896         object_name = strdup(&name[2]);
1897         if (!object_name)
1898                 return NULL;
1899
1900         func_name = strchr(object_name, '.');
1901         if (!func_name) {
1902                 free(object_name);
1903                 return NULL;
1904         }
1905
1906         *func_name = 0;
1907         func_name++;
1908
1909         object = extern_obj_find(p, object_name);
1910         if (!object) {
1911                 free(object_name);
1912                 return NULL;
1913         }
1914
1915         func = extern_type_member_func_find(object->type, func_name);
1916         if (!func) {
1917                 free(object_name);
1918                 return NULL;
1919         }
1920
1921         if (obj)
1922                 *obj = object;
1923
1924         free(object_name);
1925         return func;
1926 }
1927
1928 static struct field *
1929 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1930                                const char *name,
1931                                struct extern_obj **object)
1932 {
1933         struct extern_obj *obj;
1934         struct field *f;
1935         char *obj_name, *field_name;
1936
1937         if ((name[0] != 'e') || (name[1] != '.'))
1938                 return NULL;
1939
1940         obj_name = strdup(&name[2]);
1941         if (!obj_name)
1942                 return NULL;
1943
1944         field_name = strchr(obj_name, '.');
1945         if (!field_name) {
1946                 free(obj_name);
1947                 return NULL;
1948         }
1949
1950         *field_name = 0;
1951         field_name++;
1952
1953         obj = extern_obj_find(p, obj_name);
1954         if (!obj) {
1955                 free(obj_name);
1956                 return NULL;
1957         }
1958
1959         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1960         if (!f) {
1961                 free(obj_name);
1962                 return NULL;
1963         }
1964
1965         if (object)
1966                 *object = obj;
1967
1968         free(obj_name);
1969         return f;
1970 }
1971
1972 int
1973 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1974         const char *name,
1975         const char *mailbox_struct_type_name,
1976         rte_swx_extern_type_constructor_t constructor,
1977         rte_swx_extern_type_destructor_t destructor)
1978 {
1979         struct extern_type *elem;
1980         struct struct_type *mailbox_struct_type;
1981
1982         CHECK(p, EINVAL);
1983
1984         CHECK_NAME(name, EINVAL);
1985         CHECK(!extern_type_find(p, name), EEXIST);
1986
1987         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1988         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1989         CHECK(mailbox_struct_type, EINVAL);
1990
1991         CHECK(constructor, EINVAL);
1992         CHECK(destructor, EINVAL);
1993
1994         /* Node allocation. */
1995         elem = calloc(1, sizeof(struct extern_type));
1996         CHECK(elem, ENOMEM);
1997
1998         /* Node initialization. */
1999         strcpy(elem->name, name);
2000         elem->mailbox_struct_type = mailbox_struct_type;
2001         elem->constructor = constructor;
2002         elem->destructor = destructor;
2003         TAILQ_INIT(&elem->funcs);
2004
2005         /* Node add to tailq. */
2006         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
2007
2008         return 0;
2009 }
2010
2011 int
2012 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
2013         const char *extern_type_name,
2014         const char *name,
2015         rte_swx_extern_type_member_func_t member_func)
2016 {
2017         struct extern_type *type;
2018         struct extern_type_member_func *type_member;
2019
2020         CHECK(p, EINVAL);
2021
2022         CHECK_NAME(extern_type_name, EINVAL);
2023         type = extern_type_find(p, extern_type_name);
2024         CHECK(type, EINVAL);
2025         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
2026
2027         CHECK_NAME(name, EINVAL);
2028         CHECK(!extern_type_member_func_find(type, name), EEXIST);
2029
2030         CHECK(member_func, EINVAL);
2031
2032         /* Node allocation. */
2033         type_member = calloc(1, sizeof(struct extern_type_member_func));
2034         CHECK(type_member, ENOMEM);
2035
2036         /* Node initialization. */
2037         strcpy(type_member->name, name);
2038         type_member->func = member_func;
2039         type_member->id = type->n_funcs;
2040
2041         /* Node add to tailq. */
2042         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
2043         type->n_funcs++;
2044
2045         return 0;
2046 }
2047
2048 int
2049 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
2050                                       const char *extern_type_name,
2051                                       const char *name,
2052                                       const char *args)
2053 {
2054         struct extern_type *type;
2055         struct extern_obj *obj;
2056         void *obj_handle;
2057
2058         CHECK(p, EINVAL);
2059
2060         CHECK_NAME(extern_type_name, EINVAL);
2061         type = extern_type_find(p, extern_type_name);
2062         CHECK(type, EINVAL);
2063
2064         CHECK_NAME(name, EINVAL);
2065         CHECK(!extern_obj_find(p, name), EEXIST);
2066
2067         /* Node allocation. */
2068         obj = calloc(1, sizeof(struct extern_obj));
2069         CHECK(obj, ENOMEM);
2070
2071         /* Object construction. */
2072         obj_handle = type->constructor(args);
2073         if (!obj_handle) {
2074                 free(obj);
2075                 CHECK(0, ENODEV);
2076         }
2077
2078         /* Node initialization. */
2079         strcpy(obj->name, name);
2080         obj->type = type;
2081         obj->obj = obj_handle;
2082         obj->struct_id = p->n_structs;
2083         obj->id = p->n_extern_objs;
2084
2085         /* Node add to tailq. */
2086         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
2087         p->n_extern_objs++;
2088         p->n_structs++;
2089
2090         return 0;
2091 }
2092
2093 static int
2094 extern_obj_build(struct rte_swx_pipeline *p)
2095 {
2096         uint32_t i;
2097
2098         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2099                 struct thread *t = &p->threads[i];
2100                 struct extern_obj *obj;
2101
2102                 t->extern_objs = calloc(p->n_extern_objs,
2103                                         sizeof(struct extern_obj_runtime));
2104                 CHECK(t->extern_objs, ENOMEM);
2105
2106                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
2107                         struct extern_obj_runtime *r =
2108                                 &t->extern_objs[obj->id];
2109                         struct extern_type_member_func *func;
2110                         uint32_t mailbox_size =
2111                                 obj->type->mailbox_struct_type->n_bits / 8;
2112
2113                         r->obj = obj->obj;
2114
2115                         r->mailbox = calloc(1, mailbox_size);
2116                         CHECK(r->mailbox, ENOMEM);
2117
2118                         TAILQ_FOREACH(func, &obj->type->funcs, node)
2119                                 r->funcs[func->id] = func->func;
2120
2121                         t->structs[obj->struct_id] = r->mailbox;
2122                 }
2123         }
2124
2125         return 0;
2126 }
2127
2128 static void
2129 extern_obj_build_free(struct rte_swx_pipeline *p)
2130 {
2131         uint32_t i;
2132
2133         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2134                 struct thread *t = &p->threads[i];
2135                 uint32_t j;
2136
2137                 if (!t->extern_objs)
2138                         continue;
2139
2140                 for (j = 0; j < p->n_extern_objs; j++) {
2141                         struct extern_obj_runtime *r = &t->extern_objs[j];
2142
2143                         free(r->mailbox);
2144                 }
2145
2146                 free(t->extern_objs);
2147                 t->extern_objs = NULL;
2148         }
2149 }
2150
2151 static void
2152 extern_obj_free(struct rte_swx_pipeline *p)
2153 {
2154         extern_obj_build_free(p);
2155
2156         /* Extern objects. */
2157         for ( ; ; ) {
2158                 struct extern_obj *elem;
2159
2160                 elem = TAILQ_FIRST(&p->extern_objs);
2161                 if (!elem)
2162                         break;
2163
2164                 TAILQ_REMOVE(&p->extern_objs, elem, node);
2165                 if (elem->obj)
2166                         elem->type->destructor(elem->obj);
2167                 free(elem);
2168         }
2169
2170         /* Extern types. */
2171         for ( ; ; ) {
2172                 struct extern_type *elem;
2173
2174                 elem = TAILQ_FIRST(&p->extern_types);
2175                 if (!elem)
2176                         break;
2177
2178                 TAILQ_REMOVE(&p->extern_types, elem, node);
2179
2180                 for ( ; ; ) {
2181                         struct extern_type_member_func *func;
2182
2183                         func = TAILQ_FIRST(&elem->funcs);
2184                         if (!func)
2185                                 break;
2186
2187                         TAILQ_REMOVE(&elem->funcs, func, node);
2188                         free(func);
2189                 }
2190
2191                 free(elem);
2192         }
2193 }
2194
2195 /*
2196  * Extern function.
2197  */
2198 static struct extern_func *
2199 extern_func_find(struct rte_swx_pipeline *p, const char *name)
2200 {
2201         struct extern_func *elem;
2202
2203         TAILQ_FOREACH(elem, &p->extern_funcs, node)
2204                 if (strcmp(elem->name, name) == 0)
2205                         return elem;
2206
2207         return NULL;
2208 }
2209
2210 static struct extern_func *
2211 extern_func_parse(struct rte_swx_pipeline *p,
2212                   const char *name)
2213 {
2214         if (name[0] != 'f' || name[1] != '.')
2215                 return NULL;
2216
2217         return extern_func_find(p, &name[2]);
2218 }
2219
2220 static struct field *
2221 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
2222                                 const char *name,
2223                                 struct extern_func **function)
2224 {
2225         struct extern_func *func;
2226         struct field *f;
2227         char *func_name, *field_name;
2228
2229         if ((name[0] != 'f') || (name[1] != '.'))
2230                 return NULL;
2231
2232         func_name = strdup(&name[2]);
2233         if (!func_name)
2234                 return NULL;
2235
2236         field_name = strchr(func_name, '.');
2237         if (!field_name) {
2238                 free(func_name);
2239                 return NULL;
2240         }
2241
2242         *field_name = 0;
2243         field_name++;
2244
2245         func = extern_func_find(p, func_name);
2246         if (!func) {
2247                 free(func_name);
2248                 return NULL;
2249         }
2250
2251         f = struct_type_field_find(func->mailbox_struct_type, field_name);
2252         if (!f) {
2253                 free(func_name);
2254                 return NULL;
2255         }
2256
2257         if (function)
2258                 *function = func;
2259
2260         free(func_name);
2261         return f;
2262 }
2263
2264 int
2265 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
2266                                       const char *name,
2267                                       const char *mailbox_struct_type_name,
2268                                       rte_swx_extern_func_t func)
2269 {
2270         struct extern_func *f;
2271         struct struct_type *mailbox_struct_type;
2272
2273         CHECK(p, EINVAL);
2274
2275         CHECK_NAME(name, EINVAL);
2276         CHECK(!extern_func_find(p, name), EEXIST);
2277
2278         CHECK_NAME(mailbox_struct_type_name, EINVAL);
2279         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2280         CHECK(mailbox_struct_type, EINVAL);
2281
2282         CHECK(func, EINVAL);
2283
2284         /* Node allocation. */
2285         f = calloc(1, sizeof(struct extern_func));
2286         CHECK(func, ENOMEM);
2287
2288         /* Node initialization. */
2289         strcpy(f->name, name);
2290         f->mailbox_struct_type = mailbox_struct_type;
2291         f->func = func;
2292         f->struct_id = p->n_structs;
2293         f->id = p->n_extern_funcs;
2294
2295         /* Node add to tailq. */
2296         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
2297         p->n_extern_funcs++;
2298         p->n_structs++;
2299
2300         return 0;
2301 }
2302
2303 static int
2304 extern_func_build(struct rte_swx_pipeline *p)
2305 {
2306         uint32_t i;
2307
2308         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2309                 struct thread *t = &p->threads[i];
2310                 struct extern_func *func;
2311
2312                 /* Memory allocation. */
2313                 t->extern_funcs = calloc(p->n_extern_funcs,
2314                                          sizeof(struct extern_func_runtime));
2315                 CHECK(t->extern_funcs, ENOMEM);
2316
2317                 /* Extern function. */
2318                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
2319                         struct extern_func_runtime *r =
2320                                 &t->extern_funcs[func->id];
2321                         uint32_t mailbox_size =
2322                                 func->mailbox_struct_type->n_bits / 8;
2323
2324                         r->func = func->func;
2325
2326                         r->mailbox = calloc(1, mailbox_size);
2327                         CHECK(r->mailbox, ENOMEM);
2328
2329                         t->structs[func->struct_id] = r->mailbox;
2330                 }
2331         }
2332
2333         return 0;
2334 }
2335
2336 static void
2337 extern_func_build_free(struct rte_swx_pipeline *p)
2338 {
2339         uint32_t i;
2340
2341         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2342                 struct thread *t = &p->threads[i];
2343                 uint32_t j;
2344
2345                 if (!t->extern_funcs)
2346                         continue;
2347
2348                 for (j = 0; j < p->n_extern_funcs; j++) {
2349                         struct extern_func_runtime *r = &t->extern_funcs[j];
2350
2351                         free(r->mailbox);
2352                 }
2353
2354                 free(t->extern_funcs);
2355                 t->extern_funcs = NULL;
2356         }
2357 }
2358
2359 static void
2360 extern_func_free(struct rte_swx_pipeline *p)
2361 {
2362         extern_func_build_free(p);
2363
2364         for ( ; ; ) {
2365                 struct extern_func *elem;
2366
2367                 elem = TAILQ_FIRST(&p->extern_funcs);
2368                 if (!elem)
2369                         break;
2370
2371                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
2372                 free(elem);
2373         }
2374 }
2375
2376 /*
2377  * Header.
2378  */
2379 static struct header *
2380 header_find(struct rte_swx_pipeline *p, const char *name)
2381 {
2382         struct header *elem;
2383
2384         TAILQ_FOREACH(elem, &p->headers, node)
2385                 if (strcmp(elem->name, name) == 0)
2386                         return elem;
2387
2388         return NULL;
2389 }
2390
2391 static struct header *
2392 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
2393 {
2394         struct header *elem;
2395
2396         TAILQ_FOREACH(elem, &p->headers, node)
2397                 if (elem->struct_id == struct_id)
2398                         return elem;
2399
2400         return NULL;
2401 }
2402
2403 static struct header *
2404 header_parse(struct rte_swx_pipeline *p,
2405              const char *name)
2406 {
2407         if (name[0] != 'h' || name[1] != '.')
2408                 return NULL;
2409
2410         return header_find(p, &name[2]);
2411 }
2412
2413 static struct field *
2414 header_field_parse(struct rte_swx_pipeline *p,
2415                    const char *name,
2416                    struct header **header)
2417 {
2418         struct header *h;
2419         struct field *f;
2420         char *header_name, *field_name;
2421
2422         if ((name[0] != 'h') || (name[1] != '.'))
2423                 return NULL;
2424
2425         header_name = strdup(&name[2]);
2426         if (!header_name)
2427                 return NULL;
2428
2429         field_name = strchr(header_name, '.');
2430         if (!field_name) {
2431                 free(header_name);
2432                 return NULL;
2433         }
2434
2435         *field_name = 0;
2436         field_name++;
2437
2438         h = header_find(p, header_name);
2439         if (!h) {
2440                 free(header_name);
2441                 return NULL;
2442         }
2443
2444         f = struct_type_field_find(h->st, field_name);
2445         if (!f) {
2446                 free(header_name);
2447                 return NULL;
2448         }
2449
2450         if (header)
2451                 *header = h;
2452
2453         free(header_name);
2454         return f;
2455 }
2456
2457 int
2458 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2459                                         const char *name,
2460                                         const char *struct_type_name)
2461 {
2462         struct struct_type *st;
2463         struct header *h;
2464         size_t n_headers_max;
2465
2466         CHECK(p, EINVAL);
2467         CHECK_NAME(name, EINVAL);
2468         CHECK_NAME(struct_type_name, EINVAL);
2469
2470         CHECK(!header_find(p, name), EEXIST);
2471
2472         st = struct_type_find(p, struct_type_name);
2473         CHECK(st, EINVAL);
2474
2475         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2476         CHECK(p->n_headers < n_headers_max, ENOSPC);
2477
2478         /* Node allocation. */
2479         h = calloc(1, sizeof(struct header));
2480         CHECK(h, ENOMEM);
2481
2482         /* Node initialization. */
2483         strcpy(h->name, name);
2484         h->st = st;
2485         h->struct_id = p->n_structs;
2486         h->id = p->n_headers;
2487
2488         /* Node add to tailq. */
2489         TAILQ_INSERT_TAIL(&p->headers, h, node);
2490         p->n_headers++;
2491         p->n_structs++;
2492
2493         return 0;
2494 }
2495
2496 static int
2497 header_build(struct rte_swx_pipeline *p)
2498 {
2499         struct header *h;
2500         uint32_t n_bytes = 0, i;
2501
2502         TAILQ_FOREACH(h, &p->headers, node) {
2503                 n_bytes += h->st->n_bits / 8;
2504         }
2505
2506         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2507                 struct thread *t = &p->threads[i];
2508                 uint32_t offset = 0;
2509
2510                 t->headers = calloc(p->n_headers,
2511                                     sizeof(struct header_runtime));
2512                 CHECK(t->headers, ENOMEM);
2513
2514                 t->headers_out = calloc(p->n_headers,
2515                                         sizeof(struct header_out_runtime));
2516                 CHECK(t->headers_out, ENOMEM);
2517
2518                 t->header_storage = calloc(1, n_bytes);
2519                 CHECK(t->header_storage, ENOMEM);
2520
2521                 t->header_out_storage = calloc(1, n_bytes);
2522                 CHECK(t->header_out_storage, ENOMEM);
2523
2524                 TAILQ_FOREACH(h, &p->headers, node) {
2525                         uint8_t *header_storage;
2526                         uint32_t n_bytes =  h->st->n_bits / 8;
2527
2528                         header_storage = &t->header_storage[offset];
2529                         offset += n_bytes;
2530
2531                         t->headers[h->id].ptr0 = header_storage;
2532                         t->headers[h->id].n_bytes = n_bytes;
2533
2534                         t->structs[h->struct_id] = header_storage;
2535                 }
2536         }
2537
2538         return 0;
2539 }
2540
2541 static void
2542 header_build_free(struct rte_swx_pipeline *p)
2543 {
2544         uint32_t i;
2545
2546         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2547                 struct thread *t = &p->threads[i];
2548
2549                 free(t->headers_out);
2550                 t->headers_out = NULL;
2551
2552                 free(t->headers);
2553                 t->headers = NULL;
2554
2555                 free(t->header_out_storage);
2556                 t->header_out_storage = NULL;
2557
2558                 free(t->header_storage);
2559                 t->header_storage = NULL;
2560         }
2561 }
2562
2563 static void
2564 header_free(struct rte_swx_pipeline *p)
2565 {
2566         header_build_free(p);
2567
2568         for ( ; ; ) {
2569                 struct header *elem;
2570
2571                 elem = TAILQ_FIRST(&p->headers);
2572                 if (!elem)
2573                         break;
2574
2575                 TAILQ_REMOVE(&p->headers, elem, node);
2576                 free(elem);
2577         }
2578 }
2579
2580 /*
2581  * Meta-data.
2582  */
2583 static struct field *
2584 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2585 {
2586         if (!p->metadata_st)
2587                 return NULL;
2588
2589         if (name[0] != 'm' || name[1] != '.')
2590                 return NULL;
2591
2592         return struct_type_field_find(p->metadata_st, &name[2]);
2593 }
2594
2595 int
2596 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2597                                           const char *struct_type_name)
2598 {
2599         struct struct_type *st = NULL;
2600
2601         CHECK(p, EINVAL);
2602
2603         CHECK_NAME(struct_type_name, EINVAL);
2604         st  = struct_type_find(p, struct_type_name);
2605         CHECK(st, EINVAL);
2606         CHECK(!p->metadata_st, EINVAL);
2607
2608         p->metadata_st = st;
2609         p->metadata_struct_id = p->n_structs;
2610
2611         p->n_structs++;
2612
2613         return 0;
2614 }
2615
2616 static int
2617 metadata_build(struct rte_swx_pipeline *p)
2618 {
2619         uint32_t n_bytes = p->metadata_st->n_bits / 8;
2620         uint32_t i;
2621
2622         /* Thread-level initialization. */
2623         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2624                 struct thread *t = &p->threads[i];
2625                 uint8_t *metadata;
2626
2627                 metadata = calloc(1, n_bytes);
2628                 CHECK(metadata, ENOMEM);
2629
2630                 t->metadata = metadata;
2631                 t->structs[p->metadata_struct_id] = metadata;
2632         }
2633
2634         return 0;
2635 }
2636
2637 static void
2638 metadata_build_free(struct rte_swx_pipeline *p)
2639 {
2640         uint32_t i;
2641
2642         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2643                 struct thread *t = &p->threads[i];
2644
2645                 free(t->metadata);
2646                 t->metadata = NULL;
2647         }
2648 }
2649
2650 static void
2651 metadata_free(struct rte_swx_pipeline *p)
2652 {
2653         metadata_build_free(p);
2654 }
2655
2656 /*
2657  * Instruction.
2658  */
2659 static int
2660 instruction_is_tx(enum instruction_type type)
2661 {
2662         switch (type) {
2663         case INSTR_TX:
2664         case INSTR_TX_I:
2665                 return 1;
2666
2667         default:
2668                 return 0;
2669         }
2670 }
2671
2672 static int
2673 instruction_is_jmp(struct instruction *instr)
2674 {
2675         switch (instr->type) {
2676         case INSTR_JMP:
2677         case INSTR_JMP_VALID:
2678         case INSTR_JMP_INVALID:
2679         case INSTR_JMP_HIT:
2680         case INSTR_JMP_MISS:
2681         case INSTR_JMP_ACTION_HIT:
2682         case INSTR_JMP_ACTION_MISS:
2683         case INSTR_JMP_EQ:
2684         case INSTR_JMP_EQ_MH:
2685         case INSTR_JMP_EQ_HM:
2686         case INSTR_JMP_EQ_HH:
2687         case INSTR_JMP_EQ_I:
2688         case INSTR_JMP_NEQ:
2689         case INSTR_JMP_NEQ_MH:
2690         case INSTR_JMP_NEQ_HM:
2691         case INSTR_JMP_NEQ_HH:
2692         case INSTR_JMP_NEQ_I:
2693         case INSTR_JMP_LT:
2694         case INSTR_JMP_LT_MH:
2695         case INSTR_JMP_LT_HM:
2696         case INSTR_JMP_LT_HH:
2697         case INSTR_JMP_LT_MI:
2698         case INSTR_JMP_LT_HI:
2699         case INSTR_JMP_GT:
2700         case INSTR_JMP_GT_MH:
2701         case INSTR_JMP_GT_HM:
2702         case INSTR_JMP_GT_HH:
2703         case INSTR_JMP_GT_MI:
2704         case INSTR_JMP_GT_HI:
2705                 return 1;
2706
2707         default:
2708                 return 0;
2709         }
2710 }
2711
2712 static struct field *
2713 action_field_parse(struct action *action, const char *name);
2714
2715 static struct field *
2716 struct_field_parse(struct rte_swx_pipeline *p,
2717                    struct action *action,
2718                    const char *name,
2719                    uint32_t *struct_id)
2720 {
2721         struct field *f;
2722
2723         switch (name[0]) {
2724         case 'h':
2725         {
2726                 struct header *header;
2727
2728                 f = header_field_parse(p, name, &header);
2729                 if (!f)
2730                         return NULL;
2731
2732                 *struct_id = header->struct_id;
2733                 return f;
2734         }
2735
2736         case 'm':
2737         {
2738                 f = metadata_field_parse(p, name);
2739                 if (!f)
2740                         return NULL;
2741
2742                 *struct_id = p->metadata_struct_id;
2743                 return f;
2744         }
2745
2746         case 't':
2747         {
2748                 if (!action)
2749                         return NULL;
2750
2751                 f = action_field_parse(action, name);
2752                 if (!f)
2753                         return NULL;
2754
2755                 *struct_id = 0;
2756                 return f;
2757         }
2758
2759         case 'e':
2760         {
2761                 struct extern_obj *obj;
2762
2763                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2764                 if (!f)
2765                         return NULL;
2766
2767                 *struct_id = obj->struct_id;
2768                 return f;
2769         }
2770
2771         case 'f':
2772         {
2773                 struct extern_func *func;
2774
2775                 f = extern_func_mailbox_field_parse(p, name, &func);
2776                 if (!f)
2777                         return NULL;
2778
2779                 *struct_id = func->struct_id;
2780                 return f;
2781         }
2782
2783         default:
2784                 return NULL;
2785         }
2786 }
2787
2788 static inline void
2789 pipeline_port_inc(struct rte_swx_pipeline *p)
2790 {
2791         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2792 }
2793
2794 static inline void
2795 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2796 {
2797         t->ip = p->instructions;
2798 }
2799
2800 static inline void
2801 thread_ip_set(struct thread *t, struct instruction *ip)
2802 {
2803         t->ip = ip;
2804 }
2805
2806 static inline void
2807 thread_ip_action_call(struct rte_swx_pipeline *p,
2808                       struct thread *t,
2809                       uint32_t action_id)
2810 {
2811         t->ret = t->ip + 1;
2812         t->ip = p->action_instructions[action_id];
2813 }
2814
2815 static inline void
2816 thread_ip_inc(struct rte_swx_pipeline *p);
2817
2818 static inline void
2819 thread_ip_inc(struct rte_swx_pipeline *p)
2820 {
2821         struct thread *t = &p->threads[p->thread_id];
2822
2823         t->ip++;
2824 }
2825
2826 static inline void
2827 thread_ip_inc_cond(struct thread *t, int cond)
2828 {
2829         t->ip += cond;
2830 }
2831
2832 static inline void
2833 thread_yield(struct rte_swx_pipeline *p)
2834 {
2835         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2836 }
2837
2838 static inline void
2839 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2840 {
2841         p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2842 }
2843
2844 /*
2845  * rx.
2846  */
2847 static int
2848 instr_rx_translate(struct rte_swx_pipeline *p,
2849                    struct action *action,
2850                    char **tokens,
2851                    int n_tokens,
2852                    struct instruction *instr,
2853                    struct instruction_data *data __rte_unused)
2854 {
2855         struct field *f;
2856
2857         CHECK(!action, EINVAL);
2858         CHECK(n_tokens == 2, EINVAL);
2859
2860         f = metadata_field_parse(p, tokens[1]);
2861         CHECK(f, EINVAL);
2862
2863         instr->type = INSTR_RX;
2864         instr->io.io.offset = f->offset / 8;
2865         instr->io.io.n_bits = f->n_bits;
2866         return 0;
2867 }
2868
2869 static inline void
2870 instr_rx_exec(struct rte_swx_pipeline *p);
2871
2872 static inline void
2873 instr_rx_exec(struct rte_swx_pipeline *p)
2874 {
2875         struct thread *t = &p->threads[p->thread_id];
2876         struct instruction *ip = t->ip;
2877         struct port_in_runtime *port = &p->in[p->port_id];
2878         struct rte_swx_pkt *pkt = &t->pkt;
2879         int pkt_received;
2880
2881         /* Packet. */
2882         pkt_received = port->pkt_rx(port->obj, pkt);
2883         t->ptr = &pkt->pkt[pkt->offset];
2884         rte_prefetch0(t->ptr);
2885
2886         TRACE("[Thread %2u] rx %s from port %u\n",
2887               p->thread_id,
2888               pkt_received ? "1 pkt" : "0 pkts",
2889               p->port_id);
2890
2891         /* Headers. */
2892         t->valid_headers = 0;
2893         t->n_headers_out = 0;
2894
2895         /* Meta-data. */
2896         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2897
2898         /* Tables. */
2899         t->table_state = p->table_state;
2900
2901         /* Thread. */
2902         pipeline_port_inc(p);
2903         thread_ip_inc_cond(t, pkt_received);
2904         thread_yield(p);
2905 }
2906
2907 /*
2908  * tx.
2909  */
2910 static int
2911 instr_tx_translate(struct rte_swx_pipeline *p,
2912                    struct action *action __rte_unused,
2913                    char **tokens,
2914                    int n_tokens,
2915                    struct instruction *instr,
2916                    struct instruction_data *data __rte_unused)
2917 {
2918         char *port = tokens[1];
2919         struct field *f;
2920         uint32_t port_val;
2921
2922         CHECK(n_tokens == 2, EINVAL);
2923
2924         f = metadata_field_parse(p, port);
2925         if (f) {
2926                 instr->type = INSTR_TX;
2927                 instr->io.io.offset = f->offset / 8;
2928                 instr->io.io.n_bits = f->n_bits;
2929                 return 0;
2930         }
2931
2932         /* TX_I. */
2933         port_val = strtoul(port, &port, 0);
2934         CHECK(!port[0], EINVAL);
2935
2936         instr->type = INSTR_TX_I;
2937         instr->io.io.val = port_val;
2938         return 0;
2939 }
2940
2941 static int
2942 instr_drop_translate(struct rte_swx_pipeline *p,
2943                      struct action *action __rte_unused,
2944                      char **tokens __rte_unused,
2945                      int n_tokens,
2946                      struct instruction *instr,
2947                      struct instruction_data *data __rte_unused)
2948 {
2949         CHECK(n_tokens == 1, EINVAL);
2950
2951         /* TX_I. */
2952         instr->type = INSTR_TX_I;
2953         instr->io.io.val = p->n_ports_out - 1;
2954         return 0;
2955 }
2956
2957 static inline void
2958 emit_handler(struct thread *t)
2959 {
2960         struct header_out_runtime *h0 = &t->headers_out[0];
2961         struct header_out_runtime *h1 = &t->headers_out[1];
2962         uint32_t offset = 0, i;
2963
2964         /* No header change or header decapsulation. */
2965         if ((t->n_headers_out == 1) &&
2966             (h0->ptr + h0->n_bytes == t->ptr)) {
2967                 TRACE("Emit handler: no header change or header decap.\n");
2968
2969                 t->pkt.offset -= h0->n_bytes;
2970                 t->pkt.length += h0->n_bytes;
2971
2972                 return;
2973         }
2974
2975         /* Header encapsulation (optionally, with prior header decasulation). */
2976         if ((t->n_headers_out == 2) &&
2977             (h1->ptr + h1->n_bytes == t->ptr) &&
2978             (h0->ptr == h0->ptr0)) {
2979                 uint32_t offset;
2980
2981                 TRACE("Emit handler: header encapsulation.\n");
2982
2983                 offset = h0->n_bytes + h1->n_bytes;
2984                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2985                 t->pkt.offset -= offset;
2986                 t->pkt.length += offset;
2987
2988                 return;
2989         }
2990
2991         /* Header insertion. */
2992         /* TBD */
2993
2994         /* Header extraction. */
2995         /* TBD */
2996
2997         /* For any other case. */
2998         TRACE("Emit handler: complex case.\n");
2999
3000         for (i = 0; i < t->n_headers_out; i++) {
3001                 struct header_out_runtime *h = &t->headers_out[i];
3002
3003                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
3004                 offset += h->n_bytes;
3005         }
3006
3007         if (offset) {
3008                 memcpy(t->ptr - offset, t->header_out_storage, offset);
3009                 t->pkt.offset -= offset;
3010                 t->pkt.length += offset;
3011         }
3012 }
3013
3014 static inline void
3015 instr_tx_exec(struct rte_swx_pipeline *p);
3016
3017 static inline void
3018 instr_tx_exec(struct rte_swx_pipeline *p)
3019 {
3020         struct thread *t = &p->threads[p->thread_id];
3021         struct instruction *ip = t->ip;
3022         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
3023         struct port_out_runtime *port = &p->out[port_id];
3024         struct rte_swx_pkt *pkt = &t->pkt;
3025
3026         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
3027               p->thread_id,
3028               (uint32_t)port_id);
3029
3030         /* Headers. */
3031         emit_handler(t);
3032
3033         /* Packet. */
3034         port->pkt_tx(port->obj, pkt);
3035
3036         /* Thread. */
3037         thread_ip_reset(p, t);
3038         instr_rx_exec(p);
3039 }
3040
3041 static inline void
3042 instr_tx_i_exec(struct rte_swx_pipeline *p)
3043 {
3044         struct thread *t = &p->threads[p->thread_id];
3045         struct instruction *ip = t->ip;
3046         uint64_t port_id = ip->io.io.val;
3047         struct port_out_runtime *port = &p->out[port_id];
3048         struct rte_swx_pkt *pkt = &t->pkt;
3049
3050         TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
3051               p->thread_id,
3052               (uint32_t)port_id);
3053
3054         /* Headers. */
3055         emit_handler(t);
3056
3057         /* Packet. */
3058         port->pkt_tx(port->obj, pkt);
3059
3060         /* Thread. */
3061         thread_ip_reset(p, t);
3062         instr_rx_exec(p);
3063 }
3064
3065 /*
3066  * extract.
3067  */
3068 static int
3069 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
3070                             struct action *action,
3071                             char **tokens,
3072                             int n_tokens,
3073                             struct instruction *instr,
3074                             struct instruction_data *data __rte_unused)
3075 {
3076         struct header *h;
3077
3078         CHECK(!action, EINVAL);
3079         CHECK(n_tokens == 2, EINVAL);
3080
3081         h = header_parse(p, tokens[1]);
3082         CHECK(h, EINVAL);
3083
3084         instr->type = INSTR_HDR_EXTRACT;
3085         instr->io.hdr.header_id[0] = h->id;
3086         instr->io.hdr.struct_id[0] = h->struct_id;
3087         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3088         return 0;
3089 }
3090
3091 static inline void
3092 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
3093
3094 static inline void
3095 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
3096 {
3097         struct thread *t = &p->threads[p->thread_id];
3098         struct instruction *ip = t->ip;
3099         uint64_t valid_headers = t->valid_headers;
3100         uint8_t *ptr = t->ptr;
3101         uint32_t offset = t->pkt.offset;
3102         uint32_t length = t->pkt.length;
3103         uint32_t i;
3104
3105         for (i = 0; i < n_extract; i++) {
3106                 uint32_t header_id = ip->io.hdr.header_id[i];
3107                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3108                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3109
3110                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
3111                       p->thread_id,
3112                       header_id,
3113                       n_bytes);
3114
3115                 /* Headers. */
3116                 t->structs[struct_id] = ptr;
3117                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3118
3119                 /* Packet. */
3120                 offset += n_bytes;
3121                 length -= n_bytes;
3122                 ptr += n_bytes;
3123         }
3124
3125         /* Headers. */
3126         t->valid_headers = valid_headers;
3127
3128         /* Packet. */
3129         t->pkt.offset = offset;
3130         t->pkt.length = length;
3131         t->ptr = ptr;
3132 }
3133
3134 static inline void
3135 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
3136 {
3137         __instr_hdr_extract_exec(p, 1);
3138
3139         /* Thread. */
3140         thread_ip_inc(p);
3141 }
3142
3143 static inline void
3144 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
3145 {
3146         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3147               p->thread_id);
3148
3149         __instr_hdr_extract_exec(p, 2);
3150
3151         /* Thread. */
3152         thread_ip_inc(p);
3153 }
3154
3155 static inline void
3156 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
3157 {
3158         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3159               p->thread_id);
3160
3161         __instr_hdr_extract_exec(p, 3);
3162
3163         /* Thread. */
3164         thread_ip_inc(p);
3165 }
3166
3167 static inline void
3168 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
3169 {
3170         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3171               p->thread_id);
3172
3173         __instr_hdr_extract_exec(p, 4);
3174
3175         /* Thread. */
3176         thread_ip_inc(p);
3177 }
3178
3179 static inline void
3180 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
3181 {
3182         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3183               p->thread_id);
3184
3185         __instr_hdr_extract_exec(p, 5);
3186
3187         /* Thread. */
3188         thread_ip_inc(p);
3189 }
3190
3191 static inline void
3192 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
3193 {
3194         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3195               p->thread_id);
3196
3197         __instr_hdr_extract_exec(p, 6);
3198
3199         /* Thread. */
3200         thread_ip_inc(p);
3201 }
3202
3203 static inline void
3204 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
3205 {
3206         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3207               p->thread_id);
3208
3209         __instr_hdr_extract_exec(p, 7);
3210
3211         /* Thread. */
3212         thread_ip_inc(p);
3213 }
3214
3215 static inline void
3216 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
3217 {
3218         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3219               p->thread_id);
3220
3221         __instr_hdr_extract_exec(p, 8);
3222
3223         /* Thread. */
3224         thread_ip_inc(p);
3225 }
3226
3227 /*
3228  * emit.
3229  */
3230 static int
3231 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
3232                          struct action *action __rte_unused,
3233                          char **tokens,
3234                          int n_tokens,
3235                          struct instruction *instr,
3236                          struct instruction_data *data __rte_unused)
3237 {
3238         struct header *h;
3239
3240         CHECK(n_tokens == 2, EINVAL);
3241
3242         h = header_parse(p, tokens[1]);
3243         CHECK(h, EINVAL);
3244
3245         instr->type = INSTR_HDR_EMIT;
3246         instr->io.hdr.header_id[0] = h->id;
3247         instr->io.hdr.struct_id[0] = h->struct_id;
3248         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3249         return 0;
3250 }
3251
3252 static inline void
3253 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
3254
3255 static inline void
3256 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
3257 {
3258         struct thread *t = &p->threads[p->thread_id];
3259         struct instruction *ip = t->ip;
3260         uint64_t valid_headers = t->valid_headers;
3261         uint32_t n_headers_out = t->n_headers_out;
3262         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
3263         uint8_t *ho_ptr = NULL;
3264         uint32_t ho_nbytes = 0, first = 1, i;
3265
3266         for (i = 0; i < n_emit; i++) {
3267                 uint32_t header_id = ip->io.hdr.header_id[i];
3268                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3269
3270                 struct header_runtime *hi = &t->headers[header_id];
3271                 uint8_t *hi_ptr0 = hi->ptr0;
3272                 uint32_t n_bytes = hi->n_bytes;
3273
3274                 uint8_t *hi_ptr = t->structs[struct_id];
3275
3276                 if (!MASK64_BIT_GET(valid_headers, header_id))
3277                         continue;
3278
3279                 TRACE("[Thread %2u]: emit header %u\n",
3280                       p->thread_id,
3281                       header_id);
3282
3283                 /* Headers. */
3284                 if (first) {
3285                         first = 0;
3286
3287                         if (!t->n_headers_out) {
3288                                 ho = &t->headers_out[0];
3289
3290                                 ho->ptr0 = hi_ptr0;
3291                                 ho->ptr = hi_ptr;
3292
3293                                 ho_ptr = hi_ptr;
3294                                 ho_nbytes = n_bytes;
3295
3296                                 n_headers_out = 1;
3297
3298                                 continue;
3299                         } else {
3300                                 ho_ptr = ho->ptr;
3301                                 ho_nbytes = ho->n_bytes;
3302                         }
3303                 }
3304
3305                 if (ho_ptr + ho_nbytes == hi_ptr) {
3306                         ho_nbytes += n_bytes;
3307                 } else {
3308                         ho->n_bytes = ho_nbytes;
3309
3310                         ho++;
3311                         ho->ptr0 = hi_ptr0;
3312                         ho->ptr = hi_ptr;
3313
3314                         ho_ptr = hi_ptr;
3315                         ho_nbytes = n_bytes;
3316
3317                         n_headers_out++;
3318                 }
3319         }
3320
3321         ho->n_bytes = ho_nbytes;
3322         t->n_headers_out = n_headers_out;
3323 }
3324
3325 static inline void
3326 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
3327 {
3328         __instr_hdr_emit_exec(p, 1);
3329
3330         /* Thread. */
3331         thread_ip_inc(p);
3332 }
3333
3334 static inline void
3335 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
3336 {
3337         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3338               p->thread_id);
3339
3340         __instr_hdr_emit_exec(p, 1);
3341         instr_tx_exec(p);
3342 }
3343
3344 static inline void
3345 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
3346 {
3347         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3348               p->thread_id);
3349
3350         __instr_hdr_emit_exec(p, 2);
3351         instr_tx_exec(p);
3352 }
3353
3354 static inline void
3355 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
3356 {
3357         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3358               p->thread_id);
3359
3360         __instr_hdr_emit_exec(p, 3);
3361         instr_tx_exec(p);
3362 }
3363
3364 static inline void
3365 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
3366 {
3367         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3368               p->thread_id);
3369
3370         __instr_hdr_emit_exec(p, 4);
3371         instr_tx_exec(p);
3372 }
3373
3374 static inline void
3375 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
3376 {
3377         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3378               p->thread_id);
3379
3380         __instr_hdr_emit_exec(p, 5);
3381         instr_tx_exec(p);
3382 }
3383
3384 static inline void
3385 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
3386 {
3387         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3388               p->thread_id);
3389
3390         __instr_hdr_emit_exec(p, 6);
3391         instr_tx_exec(p);
3392 }
3393
3394 static inline void
3395 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
3396 {
3397         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3398               p->thread_id);
3399
3400         __instr_hdr_emit_exec(p, 7);
3401         instr_tx_exec(p);
3402 }
3403
3404 static inline void
3405 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
3406 {
3407         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
3408               p->thread_id);
3409
3410         __instr_hdr_emit_exec(p, 8);
3411         instr_tx_exec(p);
3412 }
3413
3414 /*
3415  * validate.
3416  */
3417 static int
3418 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
3419                              struct action *action __rte_unused,
3420                              char **tokens,
3421                              int n_tokens,
3422                              struct instruction *instr,
3423                              struct instruction_data *data __rte_unused)
3424 {
3425         struct header *h;
3426
3427         CHECK(n_tokens == 2, EINVAL);
3428
3429         h = header_parse(p, tokens[1]);
3430         CHECK(h, EINVAL);
3431
3432         instr->type = INSTR_HDR_VALIDATE;
3433         instr->valid.header_id = h->id;
3434         return 0;
3435 }
3436
3437 static inline void
3438 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
3439 {
3440         struct thread *t = &p->threads[p->thread_id];
3441         struct instruction *ip = t->ip;
3442         uint32_t header_id = ip->valid.header_id;
3443
3444         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
3445
3446         /* Headers. */
3447         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
3448
3449         /* Thread. */
3450         thread_ip_inc(p);
3451 }
3452
3453 /*
3454  * invalidate.
3455  */
3456 static int
3457 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
3458                                struct action *action __rte_unused,
3459                                char **tokens,
3460                                int n_tokens,
3461                                struct instruction *instr,
3462                                struct instruction_data *data __rte_unused)
3463 {
3464         struct header *h;
3465
3466         CHECK(n_tokens == 2, EINVAL);
3467
3468         h = header_parse(p, tokens[1]);
3469         CHECK(h, EINVAL);
3470
3471         instr->type = INSTR_HDR_INVALIDATE;
3472         instr->valid.header_id = h->id;
3473         return 0;
3474 }
3475
3476 static inline void
3477 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3478 {
3479         struct thread *t = &p->threads[p->thread_id];
3480         struct instruction *ip = t->ip;
3481         uint32_t header_id = ip->valid.header_id;
3482
3483         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3484
3485         /* Headers. */
3486         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3487
3488         /* Thread. */
3489         thread_ip_inc(p);
3490 }
3491
3492 /*
3493  * table.
3494  */
3495 static struct table *
3496 table_find(struct rte_swx_pipeline *p, const char *name);
3497
3498 static struct selector *
3499 selector_find(struct rte_swx_pipeline *p, const char *name);
3500
3501 static int
3502 instr_table_translate(struct rte_swx_pipeline *p,
3503                       struct action *action,
3504                       char **tokens,
3505                       int n_tokens,
3506                       struct instruction *instr,
3507                       struct instruction_data *data __rte_unused)
3508 {
3509         struct table *t;
3510         struct selector *s;
3511
3512         CHECK(!action, EINVAL);
3513         CHECK(n_tokens == 2, EINVAL);
3514
3515         t = table_find(p, tokens[1]);
3516         if (t) {
3517                 instr->type = INSTR_TABLE;
3518                 instr->table.table_id = t->id;
3519                 return 0;
3520         }
3521
3522         s = selector_find(p, tokens[1]);
3523         if (s) {
3524                 instr->type = INSTR_SELECTOR;
3525                 instr->table.table_id = s->id;
3526                 return 0;
3527         }
3528
3529         CHECK(0, EINVAL);
3530 }
3531
3532 static inline void
3533 instr_table_exec(struct rte_swx_pipeline *p)
3534 {
3535         struct thread *t = &p->threads[p->thread_id];
3536         struct instruction *ip = t->ip;
3537         uint32_t table_id = ip->table.table_id;
3538         struct rte_swx_table_state *ts = &t->table_state[table_id];
3539         struct table_runtime *table = &t->tables[table_id];
3540         struct table_statistics *stats = &p->table_stats[table_id];
3541         uint64_t action_id, n_pkts_hit, n_pkts_action;
3542         uint8_t *action_data;
3543         int done, hit;
3544
3545         /* Table. */
3546         done = table->func(ts->obj,
3547                            table->mailbox,
3548                            table->key,
3549                            &action_id,
3550                            &action_data,
3551                            &hit);
3552         if (!done) {
3553                 /* Thread. */
3554                 TRACE("[Thread %2u] table %u (not finalized)\n",
3555                       p->thread_id,
3556                       table_id);
3557
3558                 thread_yield(p);
3559                 return;
3560         }
3561
3562         action_id = hit ? action_id : ts->default_action_id;
3563         action_data = hit ? action_data : ts->default_action_data;
3564         n_pkts_hit = stats->n_pkts_hit[hit];
3565         n_pkts_action = stats->n_pkts_action[action_id];
3566
3567         TRACE("[Thread %2u] table %u (%s, action %u)\n",
3568               p->thread_id,
3569               table_id,
3570               hit ? "hit" : "miss",
3571               (uint32_t)action_id);
3572
3573         t->action_id = action_id;
3574         t->structs[0] = action_data;
3575         t->hit = hit;
3576         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
3577         stats->n_pkts_action[action_id] = n_pkts_action + 1;
3578
3579         /* Thread. */
3580         thread_ip_action_call(p, t, action_id);
3581 }
3582
3583 static inline void
3584 instr_selector_exec(struct rte_swx_pipeline *p)
3585 {
3586         struct thread *t = &p->threads[p->thread_id];
3587         struct instruction *ip = t->ip;
3588         uint32_t selector_id = ip->table.table_id;
3589         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
3590         struct selector_runtime *selector = &t->selectors[selector_id];
3591         struct selector_statistics *stats = &p->selector_stats[selector_id];
3592         uint64_t n_pkts = stats->n_pkts;
3593         int done;
3594
3595         /* Table. */
3596         done = rte_swx_table_selector_select(ts->obj,
3597                            selector->mailbox,
3598                            selector->group_id_buffer,
3599                            selector->selector_buffer,
3600                            selector->member_id_buffer);
3601         if (!done) {
3602                 /* Thread. */
3603                 TRACE("[Thread %2u] selector %u (not finalized)\n",
3604                       p->thread_id,
3605                       selector_id);
3606
3607                 thread_yield(p);
3608                 return;
3609         }
3610
3611
3612         TRACE("[Thread %2u] selector %u\n",
3613               p->thread_id,
3614               selector_id);
3615
3616         stats->n_pkts = n_pkts + 1;
3617
3618         /* Thread. */
3619         thread_ip_inc(p);
3620 }
3621
3622 /*
3623  * extern.
3624  */
3625 static int
3626 instr_extern_translate(struct rte_swx_pipeline *p,
3627                        struct action *action __rte_unused,
3628                        char **tokens,
3629                        int n_tokens,
3630                        struct instruction *instr,
3631                        struct instruction_data *data __rte_unused)
3632 {
3633         char *token = tokens[1];
3634
3635         CHECK(n_tokens == 2, EINVAL);
3636
3637         if (token[0] == 'e') {
3638                 struct extern_obj *obj;
3639                 struct extern_type_member_func *func;
3640
3641                 func = extern_obj_member_func_parse(p, token, &obj);
3642                 CHECK(func, EINVAL);
3643
3644                 instr->type = INSTR_EXTERN_OBJ;
3645                 instr->ext_obj.ext_obj_id = obj->id;
3646                 instr->ext_obj.func_id = func->id;
3647
3648                 return 0;
3649         }
3650
3651         if (token[0] == 'f') {
3652                 struct extern_func *func;
3653
3654                 func = extern_func_parse(p, token);
3655                 CHECK(func, EINVAL);
3656
3657                 instr->type = INSTR_EXTERN_FUNC;
3658                 instr->ext_func.ext_func_id = func->id;
3659
3660                 return 0;
3661         }
3662
3663         CHECK(0, EINVAL);
3664 }
3665
3666 static inline void
3667 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3668 {
3669         struct thread *t = &p->threads[p->thread_id];
3670         struct instruction *ip = t->ip;
3671         uint32_t obj_id = ip->ext_obj.ext_obj_id;
3672         uint32_t func_id = ip->ext_obj.func_id;
3673         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3674         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3675
3676         TRACE("[Thread %2u] extern obj %u member func %u\n",
3677               p->thread_id,
3678               obj_id,
3679               func_id);
3680
3681         /* Extern object member function execute. */
3682         uint32_t done = func(obj->obj, obj->mailbox);
3683
3684         /* Thread. */
3685         thread_ip_inc_cond(t, done);
3686         thread_yield_cond(p, done ^ 1);
3687 }
3688
3689 static inline void
3690 instr_extern_func_exec(struct rte_swx_pipeline *p)
3691 {
3692         struct thread *t = &p->threads[p->thread_id];
3693         struct instruction *ip = t->ip;
3694         uint32_t ext_func_id = ip->ext_func.ext_func_id;
3695         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3696         rte_swx_extern_func_t func = ext_func->func;
3697
3698         TRACE("[Thread %2u] extern func %u\n",
3699               p->thread_id,
3700               ext_func_id);
3701
3702         /* Extern function execute. */
3703         uint32_t done = func(ext_func->mailbox);
3704
3705         /* Thread. */
3706         thread_ip_inc_cond(t, done);
3707         thread_yield_cond(p, done ^ 1);
3708 }
3709
3710 /*
3711  * mov.
3712  */
3713 static int
3714 instr_mov_translate(struct rte_swx_pipeline *p,
3715                     struct action *action,
3716                     char **tokens,
3717                     int n_tokens,
3718                     struct instruction *instr,
3719                     struct instruction_data *data __rte_unused)
3720 {
3721         char *dst = tokens[1], *src = tokens[2];
3722         struct field *fdst, *fsrc;
3723         uint64_t src_val;
3724         uint32_t dst_struct_id = 0, src_struct_id = 0;
3725
3726         CHECK(n_tokens == 3, EINVAL);
3727
3728         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3729         CHECK(fdst, EINVAL);
3730
3731         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
3732         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3733         if (fsrc) {
3734                 instr->type = INSTR_MOV;
3735                 if (dst[0] != 'h' && src[0] == 'h')
3736                         instr->type = INSTR_MOV_MH;
3737                 if (dst[0] == 'h' && src[0] != 'h')
3738                         instr->type = INSTR_MOV_HM;
3739                 if (dst[0] == 'h' && src[0] == 'h')
3740                         instr->type = INSTR_MOV_HH;
3741
3742                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3743                 instr->mov.dst.n_bits = fdst->n_bits;
3744                 instr->mov.dst.offset = fdst->offset / 8;
3745                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3746                 instr->mov.src.n_bits = fsrc->n_bits;
3747                 instr->mov.src.offset = fsrc->offset / 8;
3748                 return 0;
3749         }
3750
3751         /* MOV_I. */
3752         src_val = strtoull(src, &src, 0);
3753         CHECK(!src[0], EINVAL);
3754
3755         if (dst[0] == 'h')
3756                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3757
3758         instr->type = INSTR_MOV_I;
3759         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3760         instr->mov.dst.n_bits = fdst->n_bits;
3761         instr->mov.dst.offset = fdst->offset / 8;
3762         instr->mov.src_val = src_val;
3763         return 0;
3764 }
3765
3766 static inline void
3767 instr_mov_exec(struct rte_swx_pipeline *p)
3768 {
3769         struct thread *t = &p->threads[p->thread_id];
3770         struct instruction *ip = t->ip;
3771
3772         TRACE("[Thread %2u] mov\n",
3773               p->thread_id);
3774
3775         MOV(t, ip);
3776
3777         /* Thread. */
3778         thread_ip_inc(p);
3779 }
3780
3781 static inline void
3782 instr_mov_mh_exec(struct rte_swx_pipeline *p)
3783 {
3784         struct thread *t = &p->threads[p->thread_id];
3785         struct instruction *ip = t->ip;
3786
3787         TRACE("[Thread %2u] mov (mh)\n",
3788               p->thread_id);
3789
3790         MOV_MH(t, ip);
3791
3792         /* Thread. */
3793         thread_ip_inc(p);
3794 }
3795
3796 static inline void
3797 instr_mov_hm_exec(struct rte_swx_pipeline *p)
3798 {
3799         struct thread *t = &p->threads[p->thread_id];
3800         struct instruction *ip = t->ip;
3801
3802         TRACE("[Thread %2u] mov (hm)\n",
3803               p->thread_id);
3804
3805         MOV_HM(t, ip);
3806
3807         /* Thread. */
3808         thread_ip_inc(p);
3809 }
3810
3811 static inline void
3812 instr_mov_hh_exec(struct rte_swx_pipeline *p)
3813 {
3814         struct thread *t = &p->threads[p->thread_id];
3815         struct instruction *ip = t->ip;
3816
3817         TRACE("[Thread %2u] mov (hh)\n",
3818               p->thread_id);
3819
3820         MOV_HH(t, ip);
3821
3822         /* Thread. */
3823         thread_ip_inc(p);
3824 }
3825
3826 static inline void
3827 instr_mov_i_exec(struct rte_swx_pipeline *p)
3828 {
3829         struct thread *t = &p->threads[p->thread_id];
3830         struct instruction *ip = t->ip;
3831
3832         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
3833               p->thread_id,
3834               ip->mov.src_val);
3835
3836         MOV_I(t, ip);
3837
3838         /* Thread. */
3839         thread_ip_inc(p);
3840 }
3841
3842 /*
3843  * dma.
3844  */
3845 static inline void
3846 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3847
3848 static inline void
3849 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3850 {
3851         struct thread *t = &p->threads[p->thread_id];
3852         struct instruction *ip = t->ip;
3853         uint8_t *action_data = t->structs[0];
3854         uint64_t valid_headers = t->valid_headers;
3855         uint32_t i;
3856
3857         for (i = 0; i < n_dma; i++) {
3858                 uint32_t header_id = ip->dma.dst.header_id[i];
3859                 uint32_t struct_id = ip->dma.dst.struct_id[i];
3860                 uint32_t offset = ip->dma.src.offset[i];
3861                 uint32_t n_bytes = ip->dma.n_bytes[i];
3862
3863                 struct header_runtime *h = &t->headers[header_id];
3864                 uint8_t *h_ptr0 = h->ptr0;
3865                 uint8_t *h_ptr = t->structs[struct_id];
3866
3867                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3868                         h_ptr : h_ptr0;
3869                 void *src = &action_data[offset];
3870
3871                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3872
3873                 /* Headers. */
3874                 memcpy(dst, src, n_bytes);
3875                 t->structs[struct_id] = dst;
3876                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3877         }
3878
3879         t->valid_headers = valid_headers;
3880 }
3881
3882 static inline void
3883 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3884 {
3885         __instr_dma_ht_exec(p, 1);
3886
3887         /* Thread. */
3888         thread_ip_inc(p);
3889 }
3890
3891 static inline void
3892 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3893 {
3894         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3895               p->thread_id);
3896
3897         __instr_dma_ht_exec(p, 2);
3898
3899         /* Thread. */
3900         thread_ip_inc(p);
3901 }
3902
3903 static inline void
3904 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3905 {
3906         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3907               p->thread_id);
3908
3909         __instr_dma_ht_exec(p, 3);
3910
3911         /* Thread. */
3912         thread_ip_inc(p);
3913 }
3914
3915 static inline void
3916 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3917 {
3918         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3919               p->thread_id);
3920
3921         __instr_dma_ht_exec(p, 4);
3922
3923         /* Thread. */
3924         thread_ip_inc(p);
3925 }
3926
3927 static inline void
3928 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3929 {
3930         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3931               p->thread_id);
3932
3933         __instr_dma_ht_exec(p, 5);
3934
3935         /* Thread. */
3936         thread_ip_inc(p);
3937 }
3938
3939 static inline void
3940 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3941 {
3942         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3943               p->thread_id);
3944
3945         __instr_dma_ht_exec(p, 6);
3946
3947         /* Thread. */
3948         thread_ip_inc(p);
3949 }
3950
3951 static inline void
3952 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3953 {
3954         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3955               p->thread_id);
3956
3957         __instr_dma_ht_exec(p, 7);
3958
3959         /* Thread. */
3960         thread_ip_inc(p);
3961 }
3962
3963 static inline void
3964 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3965 {
3966         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3967               p->thread_id);
3968
3969         __instr_dma_ht_exec(p, 8);
3970
3971         /* Thread. */
3972         thread_ip_inc(p);
3973 }
3974
3975 /*
3976  * alu.
3977  */
3978 static int
3979 instr_alu_add_translate(struct rte_swx_pipeline *p,
3980                         struct action *action,
3981                         char **tokens,
3982                         int n_tokens,
3983                         struct instruction *instr,
3984                         struct instruction_data *data __rte_unused)
3985 {
3986         char *dst = tokens[1], *src = tokens[2];
3987         struct field *fdst, *fsrc;
3988         uint64_t src_val;
3989         uint32_t dst_struct_id = 0, src_struct_id = 0;
3990
3991         CHECK(n_tokens == 3, EINVAL);
3992
3993         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3994         CHECK(fdst, EINVAL);
3995
3996         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3997         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3998         if (fsrc) {
3999                 instr->type = INSTR_ALU_ADD;
4000                 if (dst[0] == 'h' && src[0] != 'h')
4001                         instr->type = INSTR_ALU_ADD_HM;
4002                 if (dst[0] != 'h' && src[0] == 'h')
4003                         instr->type = INSTR_ALU_ADD_MH;
4004                 if (dst[0] == 'h' && src[0] == 'h')
4005                         instr->type = INSTR_ALU_ADD_HH;
4006
4007                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4008                 instr->alu.dst.n_bits = fdst->n_bits;
4009                 instr->alu.dst.offset = fdst->offset / 8;
4010                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4011                 instr->alu.src.n_bits = fsrc->n_bits;
4012                 instr->alu.src.offset = fsrc->offset / 8;
4013                 return 0;
4014         }
4015
4016         /* ADD_MI, ADD_HI. */
4017         src_val = strtoull(src, &src, 0);
4018         CHECK(!src[0], EINVAL);
4019
4020         instr->type = INSTR_ALU_ADD_MI;
4021         if (dst[0] == 'h')
4022                 instr->type = INSTR_ALU_ADD_HI;
4023
4024         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4025         instr->alu.dst.n_bits = fdst->n_bits;
4026         instr->alu.dst.offset = fdst->offset / 8;
4027         instr->alu.src_val = src_val;
4028         return 0;
4029 }
4030
4031 static int
4032 instr_alu_sub_translate(struct rte_swx_pipeline *p,
4033                         struct action *action,
4034                         char **tokens,
4035                         int n_tokens,
4036                         struct instruction *instr,
4037                         struct instruction_data *data __rte_unused)
4038 {
4039         char *dst = tokens[1], *src = tokens[2];
4040         struct field *fdst, *fsrc;
4041         uint64_t src_val;
4042         uint32_t dst_struct_id = 0, src_struct_id = 0;
4043
4044         CHECK(n_tokens == 3, EINVAL);
4045
4046         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4047         CHECK(fdst, EINVAL);
4048
4049         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
4050         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4051         if (fsrc) {
4052                 instr->type = INSTR_ALU_SUB;
4053                 if (dst[0] == 'h' && src[0] != 'h')
4054                         instr->type = INSTR_ALU_SUB_HM;
4055                 if (dst[0] != 'h' && src[0] == 'h')
4056                         instr->type = INSTR_ALU_SUB_MH;
4057                 if (dst[0] == 'h' && src[0] == 'h')
4058                         instr->type = INSTR_ALU_SUB_HH;
4059
4060                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4061                 instr->alu.dst.n_bits = fdst->n_bits;
4062                 instr->alu.dst.offset = fdst->offset / 8;
4063                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4064                 instr->alu.src.n_bits = fsrc->n_bits;
4065                 instr->alu.src.offset = fsrc->offset / 8;
4066                 return 0;
4067         }
4068
4069         /* SUB_MI, SUB_HI. */
4070         src_val = strtoull(src, &src, 0);
4071         CHECK(!src[0], EINVAL);
4072
4073         instr->type = INSTR_ALU_SUB_MI;
4074         if (dst[0] == 'h')
4075                 instr->type = INSTR_ALU_SUB_HI;
4076
4077         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4078         instr->alu.dst.n_bits = fdst->n_bits;
4079         instr->alu.dst.offset = fdst->offset / 8;
4080         instr->alu.src_val = src_val;
4081         return 0;
4082 }
4083
4084 static int
4085 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
4086                           struct action *action __rte_unused,
4087                           char **tokens,
4088                           int n_tokens,
4089                           struct instruction *instr,
4090                           struct instruction_data *data __rte_unused)
4091 {
4092         char *dst = tokens[1], *src = tokens[2];
4093         struct header *hdst, *hsrc;
4094         struct field *fdst, *fsrc;
4095
4096         CHECK(n_tokens == 3, EINVAL);
4097
4098         fdst = header_field_parse(p, dst, &hdst);
4099         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4100
4101         /* CKADD_FIELD. */
4102         fsrc = header_field_parse(p, src, &hsrc);
4103         if (fsrc) {
4104                 instr->type = INSTR_ALU_CKADD_FIELD;
4105                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4106                 instr->alu.dst.n_bits = fdst->n_bits;
4107                 instr->alu.dst.offset = fdst->offset / 8;
4108                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4109                 instr->alu.src.n_bits = fsrc->n_bits;
4110                 instr->alu.src.offset = fsrc->offset / 8;
4111                 return 0;
4112         }
4113
4114         /* CKADD_STRUCT, CKADD_STRUCT20. */
4115         hsrc = header_parse(p, src);
4116         CHECK(hsrc, EINVAL);
4117
4118         instr->type = INSTR_ALU_CKADD_STRUCT;
4119         if ((hsrc->st->n_bits / 8) == 20)
4120                 instr->type = INSTR_ALU_CKADD_STRUCT20;
4121
4122         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4123         instr->alu.dst.n_bits = fdst->n_bits;
4124         instr->alu.dst.offset = fdst->offset / 8;
4125         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4126         instr->alu.src.n_bits = hsrc->st->n_bits;
4127         instr->alu.src.offset = 0; /* Unused. */
4128         return 0;
4129 }
4130
4131 static int
4132 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
4133                           struct action *action __rte_unused,
4134                           char **tokens,
4135                           int n_tokens,
4136                           struct instruction *instr,
4137                           struct instruction_data *data __rte_unused)
4138 {
4139         char *dst = tokens[1], *src = tokens[2];
4140         struct header *hdst, *hsrc;
4141         struct field *fdst, *fsrc;
4142
4143         CHECK(n_tokens == 3, EINVAL);
4144
4145         fdst = header_field_parse(p, dst, &hdst);
4146         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4147
4148         fsrc = header_field_parse(p, src, &hsrc);
4149         CHECK(fsrc, EINVAL);
4150
4151         instr->type = INSTR_ALU_CKSUB_FIELD;
4152         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4153         instr->alu.dst.n_bits = fdst->n_bits;
4154         instr->alu.dst.offset = fdst->offset / 8;
4155         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4156         instr->alu.src.n_bits = fsrc->n_bits;
4157         instr->alu.src.offset = fsrc->offset / 8;
4158         return 0;
4159 }
4160
4161 static int
4162 instr_alu_shl_translate(struct rte_swx_pipeline *p,
4163                         struct action *action,
4164                         char **tokens,
4165                         int n_tokens,
4166                         struct instruction *instr,
4167                         struct instruction_data *data __rte_unused)
4168 {
4169         char *dst = tokens[1], *src = tokens[2];
4170         struct field *fdst, *fsrc;
4171         uint64_t src_val;
4172         uint32_t dst_struct_id = 0, src_struct_id = 0;
4173
4174         CHECK(n_tokens == 3, EINVAL);
4175
4176         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4177         CHECK(fdst, EINVAL);
4178
4179         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
4180         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4181         if (fsrc) {
4182                 instr->type = INSTR_ALU_SHL;
4183                 if (dst[0] == 'h' && src[0] != 'h')
4184                         instr->type = INSTR_ALU_SHL_HM;
4185                 if (dst[0] != 'h' && src[0] == 'h')
4186                         instr->type = INSTR_ALU_SHL_MH;
4187                 if (dst[0] == 'h' && src[0] == 'h')
4188                         instr->type = INSTR_ALU_SHL_HH;
4189
4190                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4191                 instr->alu.dst.n_bits = fdst->n_bits;
4192                 instr->alu.dst.offset = fdst->offset / 8;
4193                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4194                 instr->alu.src.n_bits = fsrc->n_bits;
4195                 instr->alu.src.offset = fsrc->offset / 8;
4196                 return 0;
4197         }
4198
4199         /* SHL_MI, SHL_HI. */
4200         src_val = strtoull(src, &src, 0);
4201         CHECK(!src[0], EINVAL);
4202
4203         instr->type = INSTR_ALU_SHL_MI;
4204         if (dst[0] == 'h')
4205                 instr->type = INSTR_ALU_SHL_HI;
4206
4207         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4208         instr->alu.dst.n_bits = fdst->n_bits;
4209         instr->alu.dst.offset = fdst->offset / 8;
4210         instr->alu.src_val = src_val;
4211         return 0;
4212 }
4213
4214 static int
4215 instr_alu_shr_translate(struct rte_swx_pipeline *p,
4216                         struct action *action,
4217                         char **tokens,
4218                         int n_tokens,
4219                         struct instruction *instr,
4220                         struct instruction_data *data __rte_unused)
4221 {
4222         char *dst = tokens[1], *src = tokens[2];
4223         struct field *fdst, *fsrc;
4224         uint64_t src_val;
4225         uint32_t dst_struct_id = 0, src_struct_id = 0;
4226
4227         CHECK(n_tokens == 3, EINVAL);
4228
4229         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4230         CHECK(fdst, EINVAL);
4231
4232         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
4233         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4234         if (fsrc) {
4235                 instr->type = INSTR_ALU_SHR;
4236                 if (dst[0] == 'h' && src[0] != 'h')
4237                         instr->type = INSTR_ALU_SHR_HM;
4238                 if (dst[0] != 'h' && src[0] == 'h')
4239                         instr->type = INSTR_ALU_SHR_MH;
4240                 if (dst[0] == 'h' && src[0] == 'h')
4241                         instr->type = INSTR_ALU_SHR_HH;
4242
4243                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4244                 instr->alu.dst.n_bits = fdst->n_bits;
4245                 instr->alu.dst.offset = fdst->offset / 8;
4246                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4247                 instr->alu.src.n_bits = fsrc->n_bits;
4248                 instr->alu.src.offset = fsrc->offset / 8;
4249                 return 0;
4250         }
4251
4252         /* SHR_MI, SHR_HI. */
4253         src_val = strtoull(src, &src, 0);
4254         CHECK(!src[0], EINVAL);
4255
4256         instr->type = INSTR_ALU_SHR_MI;
4257         if (dst[0] == 'h')
4258                 instr->type = INSTR_ALU_SHR_HI;
4259
4260         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4261         instr->alu.dst.n_bits = fdst->n_bits;
4262         instr->alu.dst.offset = fdst->offset / 8;
4263         instr->alu.src_val = src_val;
4264         return 0;
4265 }
4266
4267 static int
4268 instr_alu_and_translate(struct rte_swx_pipeline *p,
4269                         struct action *action,
4270                         char **tokens,
4271                         int n_tokens,
4272                         struct instruction *instr,
4273                         struct instruction_data *data __rte_unused)
4274 {
4275         char *dst = tokens[1], *src = tokens[2];
4276         struct field *fdst, *fsrc;
4277         uint64_t src_val;
4278         uint32_t dst_struct_id = 0, src_struct_id = 0;
4279
4280         CHECK(n_tokens == 3, EINVAL);
4281
4282         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4283         CHECK(fdst, EINVAL);
4284
4285         /* AND, AND_MH, AND_HM, AND_HH. */
4286         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4287         if (fsrc) {
4288                 instr->type = INSTR_ALU_AND;
4289                 if (dst[0] != 'h' && src[0] == 'h')
4290                         instr->type = INSTR_ALU_AND_MH;
4291                 if (dst[0] == 'h' && src[0] != 'h')
4292                         instr->type = INSTR_ALU_AND_HM;
4293                 if (dst[0] == 'h' && src[0] == 'h')
4294                         instr->type = INSTR_ALU_AND_HH;
4295
4296                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4297                 instr->alu.dst.n_bits = fdst->n_bits;
4298                 instr->alu.dst.offset = fdst->offset / 8;
4299                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4300                 instr->alu.src.n_bits = fsrc->n_bits;
4301                 instr->alu.src.offset = fsrc->offset / 8;
4302                 return 0;
4303         }
4304
4305         /* AND_I. */
4306         src_val = strtoull(src, &src, 0);
4307         CHECK(!src[0], EINVAL);
4308
4309         if (dst[0] == 'h')
4310                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4311
4312         instr->type = INSTR_ALU_AND_I;
4313         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4314         instr->alu.dst.n_bits = fdst->n_bits;
4315         instr->alu.dst.offset = fdst->offset / 8;
4316         instr->alu.src_val = src_val;
4317         return 0;
4318 }
4319
4320 static int
4321 instr_alu_or_translate(struct rte_swx_pipeline *p,
4322                        struct action *action,
4323                        char **tokens,
4324                        int n_tokens,
4325                        struct instruction *instr,
4326                        struct instruction_data *data __rte_unused)
4327 {
4328         char *dst = tokens[1], *src = tokens[2];
4329         struct field *fdst, *fsrc;
4330         uint64_t src_val;
4331         uint32_t dst_struct_id = 0, src_struct_id = 0;
4332
4333         CHECK(n_tokens == 3, EINVAL);
4334
4335         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4336         CHECK(fdst, EINVAL);
4337
4338         /* OR, OR_MH, OR_HM, OR_HH. */
4339         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4340         if (fsrc) {
4341                 instr->type = INSTR_ALU_OR;
4342                 if (dst[0] != 'h' && src[0] == 'h')
4343                         instr->type = INSTR_ALU_OR_MH;
4344                 if (dst[0] == 'h' && src[0] != 'h')
4345                         instr->type = INSTR_ALU_OR_HM;
4346                 if (dst[0] == 'h' && src[0] == 'h')
4347                         instr->type = INSTR_ALU_OR_HH;
4348
4349                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4350                 instr->alu.dst.n_bits = fdst->n_bits;
4351                 instr->alu.dst.offset = fdst->offset / 8;
4352                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4353                 instr->alu.src.n_bits = fsrc->n_bits;
4354                 instr->alu.src.offset = fsrc->offset / 8;
4355                 return 0;
4356         }
4357
4358         /* OR_I. */
4359         src_val = strtoull(src, &src, 0);
4360         CHECK(!src[0], EINVAL);
4361
4362         if (dst[0] == 'h')
4363                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4364
4365         instr->type = INSTR_ALU_OR_I;
4366         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4367         instr->alu.dst.n_bits = fdst->n_bits;
4368         instr->alu.dst.offset = fdst->offset / 8;
4369         instr->alu.src_val = src_val;
4370         return 0;
4371 }
4372
4373 static int
4374 instr_alu_xor_translate(struct rte_swx_pipeline *p,
4375                         struct action *action,
4376                         char **tokens,
4377                         int n_tokens,
4378                         struct instruction *instr,
4379                         struct instruction_data *data __rte_unused)
4380 {
4381         char *dst = tokens[1], *src = tokens[2];
4382         struct field *fdst, *fsrc;
4383         uint64_t src_val;
4384         uint32_t dst_struct_id = 0, src_struct_id = 0;
4385
4386         CHECK(n_tokens == 3, EINVAL);
4387
4388         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4389         CHECK(fdst, EINVAL);
4390
4391         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
4392         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4393         if (fsrc) {
4394                 instr->type = INSTR_ALU_XOR;
4395                 if (dst[0] != 'h' && src[0] == 'h')
4396                         instr->type = INSTR_ALU_XOR_MH;
4397                 if (dst[0] == 'h' && src[0] != 'h')
4398                         instr->type = INSTR_ALU_XOR_HM;
4399                 if (dst[0] == 'h' && src[0] == 'h')
4400                         instr->type = INSTR_ALU_XOR_HH;
4401
4402                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4403                 instr->alu.dst.n_bits = fdst->n_bits;
4404                 instr->alu.dst.offset = fdst->offset / 8;
4405                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4406                 instr->alu.src.n_bits = fsrc->n_bits;
4407                 instr->alu.src.offset = fsrc->offset / 8;
4408                 return 0;
4409         }
4410
4411         /* XOR_I. */
4412         src_val = strtoull(src, &src, 0);
4413         CHECK(!src[0], EINVAL);
4414
4415         if (dst[0] == 'h')
4416                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4417
4418         instr->type = INSTR_ALU_XOR_I;
4419         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4420         instr->alu.dst.n_bits = fdst->n_bits;
4421         instr->alu.dst.offset = fdst->offset / 8;
4422         instr->alu.src_val = src_val;
4423         return 0;
4424 }
4425
4426 static inline void
4427 instr_alu_add_exec(struct rte_swx_pipeline *p)
4428 {
4429         struct thread *t = &p->threads[p->thread_id];
4430         struct instruction *ip = t->ip;
4431
4432         TRACE("[Thread %2u] add\n", p->thread_id);
4433
4434         /* Structs. */
4435         ALU(t, ip, +);
4436
4437         /* Thread. */
4438         thread_ip_inc(p);
4439 }
4440
4441 static inline void
4442 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
4443 {
4444         struct thread *t = &p->threads[p->thread_id];
4445         struct instruction *ip = t->ip;
4446
4447         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
4448
4449         /* Structs. */
4450         ALU_MH(t, ip, +);
4451
4452         /* Thread. */
4453         thread_ip_inc(p);
4454 }
4455
4456 static inline void
4457 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
4458 {
4459         struct thread *t = &p->threads[p->thread_id];
4460         struct instruction *ip = t->ip;
4461
4462         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
4463
4464         /* Structs. */
4465         ALU_HM(t, ip, +);
4466
4467         /* Thread. */
4468         thread_ip_inc(p);
4469 }
4470
4471 static inline void
4472 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
4473 {
4474         struct thread *t = &p->threads[p->thread_id];
4475         struct instruction *ip = t->ip;
4476
4477         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
4478
4479         /* Structs. */
4480         ALU_HH(t, ip, +);
4481
4482         /* Thread. */
4483         thread_ip_inc(p);
4484 }
4485
4486 static inline void
4487 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
4488 {
4489         struct thread *t = &p->threads[p->thread_id];
4490         struct instruction *ip = t->ip;
4491
4492         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
4493
4494         /* Structs. */
4495         ALU_MI(t, ip, +);
4496
4497         /* Thread. */
4498         thread_ip_inc(p);
4499 }
4500
4501 static inline void
4502 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
4503 {
4504         struct thread *t = &p->threads[p->thread_id];
4505         struct instruction *ip = t->ip;
4506
4507         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
4508
4509         /* Structs. */
4510         ALU_HI(t, ip, +);
4511
4512         /* Thread. */
4513         thread_ip_inc(p);
4514 }
4515
4516 static inline void
4517 instr_alu_sub_exec(struct rte_swx_pipeline *p)
4518 {
4519         struct thread *t = &p->threads[p->thread_id];
4520         struct instruction *ip = t->ip;
4521
4522         TRACE("[Thread %2u] sub\n", p->thread_id);
4523
4524         /* Structs. */
4525         ALU(t, ip, -);
4526
4527         /* Thread. */
4528         thread_ip_inc(p);
4529 }
4530
4531 static inline void
4532 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
4533 {
4534         struct thread *t = &p->threads[p->thread_id];
4535         struct instruction *ip = t->ip;
4536
4537         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
4538
4539         /* Structs. */
4540         ALU_MH(t, ip, -);
4541
4542         /* Thread. */
4543         thread_ip_inc(p);
4544 }
4545
4546 static inline void
4547 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4548 {
4549         struct thread *t = &p->threads[p->thread_id];
4550         struct instruction *ip = t->ip;
4551
4552         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4553
4554         /* Structs. */
4555         ALU_HM(t, ip, -);
4556
4557         /* Thread. */
4558         thread_ip_inc(p);
4559 }
4560
4561 static inline void
4562 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4563 {
4564         struct thread *t = &p->threads[p->thread_id];
4565         struct instruction *ip = t->ip;
4566
4567         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4568
4569         /* Structs. */
4570         ALU_HH(t, ip, -);
4571
4572         /* Thread. */
4573         thread_ip_inc(p);
4574 }
4575
4576 static inline void
4577 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4578 {
4579         struct thread *t = &p->threads[p->thread_id];
4580         struct instruction *ip = t->ip;
4581
4582         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4583
4584         /* Structs. */
4585         ALU_MI(t, ip, -);
4586
4587         /* Thread. */
4588         thread_ip_inc(p);
4589 }
4590
4591 static inline void
4592 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4593 {
4594         struct thread *t = &p->threads[p->thread_id];
4595         struct instruction *ip = t->ip;
4596
4597         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4598
4599         /* Structs. */
4600         ALU_HI(t, ip, -);
4601
4602         /* Thread. */
4603         thread_ip_inc(p);
4604 }
4605
4606 static inline void
4607 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4608 {
4609         struct thread *t = &p->threads[p->thread_id];
4610         struct instruction *ip = t->ip;
4611
4612         TRACE("[Thread %2u] shl\n", p->thread_id);
4613
4614         /* Structs. */
4615         ALU(t, ip, <<);
4616
4617         /* Thread. */
4618         thread_ip_inc(p);
4619 }
4620
4621 static inline void
4622 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4623 {
4624         struct thread *t = &p->threads[p->thread_id];
4625         struct instruction *ip = t->ip;
4626
4627         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4628
4629         /* Structs. */
4630         ALU_MH(t, ip, <<);
4631
4632         /* Thread. */
4633         thread_ip_inc(p);
4634 }
4635
4636 static inline void
4637 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4638 {
4639         struct thread *t = &p->threads[p->thread_id];
4640         struct instruction *ip = t->ip;
4641
4642         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4643
4644         /* Structs. */
4645         ALU_HM(t, ip, <<);
4646
4647         /* Thread. */
4648         thread_ip_inc(p);
4649 }
4650
4651 static inline void
4652 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4653 {
4654         struct thread *t = &p->threads[p->thread_id];
4655         struct instruction *ip = t->ip;
4656
4657         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4658
4659         /* Structs. */
4660         ALU_HH(t, ip, <<);
4661
4662         /* Thread. */
4663         thread_ip_inc(p);
4664 }
4665
4666 static inline void
4667 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4668 {
4669         struct thread *t = &p->threads[p->thread_id];
4670         struct instruction *ip = t->ip;
4671
4672         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4673
4674         /* Structs. */
4675         ALU_MI(t, ip, <<);
4676
4677         /* Thread. */
4678         thread_ip_inc(p);
4679 }
4680
4681 static inline void
4682 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4683 {
4684         struct thread *t = &p->threads[p->thread_id];
4685         struct instruction *ip = t->ip;
4686
4687         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4688
4689         /* Structs. */
4690         ALU_HI(t, ip, <<);
4691
4692         /* Thread. */
4693         thread_ip_inc(p);
4694 }
4695
4696 static inline void
4697 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4698 {
4699         struct thread *t = &p->threads[p->thread_id];
4700         struct instruction *ip = t->ip;
4701
4702         TRACE("[Thread %2u] shr\n", p->thread_id);
4703
4704         /* Structs. */
4705         ALU(t, ip, >>);
4706
4707         /* Thread. */
4708         thread_ip_inc(p);
4709 }
4710
4711 static inline void
4712 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4713 {
4714         struct thread *t = &p->threads[p->thread_id];
4715         struct instruction *ip = t->ip;
4716
4717         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4718
4719         /* Structs. */
4720         ALU_MH(t, ip, >>);
4721
4722         /* Thread. */
4723         thread_ip_inc(p);
4724 }
4725
4726 static inline void
4727 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4728 {
4729         struct thread *t = &p->threads[p->thread_id];
4730         struct instruction *ip = t->ip;
4731
4732         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4733
4734         /* Structs. */
4735         ALU_HM(t, ip, >>);
4736
4737         /* Thread. */
4738         thread_ip_inc(p);
4739 }
4740
4741 static inline void
4742 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4743 {
4744         struct thread *t = &p->threads[p->thread_id];
4745         struct instruction *ip = t->ip;
4746
4747         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4748
4749         /* Structs. */
4750         ALU_HH(t, ip, >>);
4751
4752         /* Thread. */
4753         thread_ip_inc(p);
4754 }
4755
4756 static inline void
4757 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4758 {
4759         struct thread *t = &p->threads[p->thread_id];
4760         struct instruction *ip = t->ip;
4761
4762         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4763
4764         /* Structs. */
4765         ALU_MI(t, ip, >>);
4766
4767         /* Thread. */
4768         thread_ip_inc(p);
4769 }
4770
4771 static inline void
4772 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4773 {
4774         struct thread *t = &p->threads[p->thread_id];
4775         struct instruction *ip = t->ip;
4776
4777         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4778
4779         /* Structs. */
4780         ALU_HI(t, ip, >>);
4781
4782         /* Thread. */
4783         thread_ip_inc(p);
4784 }
4785
4786 static inline void
4787 instr_alu_and_exec(struct rte_swx_pipeline *p)
4788 {
4789         struct thread *t = &p->threads[p->thread_id];
4790         struct instruction *ip = t->ip;
4791
4792         TRACE("[Thread %2u] and\n", p->thread_id);
4793
4794         /* Structs. */
4795         ALU(t, ip, &);
4796
4797         /* Thread. */
4798         thread_ip_inc(p);
4799 }
4800
4801 static inline void
4802 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
4803 {
4804         struct thread *t = &p->threads[p->thread_id];
4805         struct instruction *ip = t->ip;
4806
4807         TRACE("[Thread %2u] and (mh)\n", p->thread_id);
4808
4809         /* Structs. */
4810         ALU_MH(t, ip, &);
4811
4812         /* Thread. */
4813         thread_ip_inc(p);
4814 }
4815
4816 static inline void
4817 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
4818 {
4819         struct thread *t = &p->threads[p->thread_id];
4820         struct instruction *ip = t->ip;
4821
4822         TRACE("[Thread %2u] and (hm)\n", p->thread_id);
4823
4824         /* Structs. */
4825         ALU_HM_FAST(t, ip, &);
4826
4827         /* Thread. */
4828         thread_ip_inc(p);
4829 }
4830
4831 static inline void
4832 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
4833 {
4834         struct thread *t = &p->threads[p->thread_id];
4835         struct instruction *ip = t->ip;
4836
4837         TRACE("[Thread %2u] and (hh)\n", p->thread_id);
4838
4839         /* Structs. */
4840         ALU_HH_FAST(t, ip, &);
4841
4842         /* Thread. */
4843         thread_ip_inc(p);
4844 }
4845
4846 static inline void
4847 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4848 {
4849         struct thread *t = &p->threads[p->thread_id];
4850         struct instruction *ip = t->ip;
4851
4852         TRACE("[Thread %2u] and (i)\n", p->thread_id);
4853
4854         /* Structs. */
4855         ALU_I(t, ip, &);
4856
4857         /* Thread. */
4858         thread_ip_inc(p);
4859 }
4860
4861 static inline void
4862 instr_alu_or_exec(struct rte_swx_pipeline *p)
4863 {
4864         struct thread *t = &p->threads[p->thread_id];
4865         struct instruction *ip = t->ip;
4866
4867         TRACE("[Thread %2u] or\n", p->thread_id);
4868
4869         /* Structs. */
4870         ALU(t, ip, |);
4871
4872         /* Thread. */
4873         thread_ip_inc(p);
4874 }
4875
4876 static inline void
4877 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
4878 {
4879         struct thread *t = &p->threads[p->thread_id];
4880         struct instruction *ip = t->ip;
4881
4882         TRACE("[Thread %2u] or (mh)\n", p->thread_id);
4883
4884         /* Structs. */
4885         ALU_MH(t, ip, |);
4886
4887         /* Thread. */
4888         thread_ip_inc(p);
4889 }
4890
4891 static inline void
4892 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
4893 {
4894         struct thread *t = &p->threads[p->thread_id];
4895         struct instruction *ip = t->ip;
4896
4897         TRACE("[Thread %2u] or (hm)\n", p->thread_id);
4898
4899         /* Structs. */
4900         ALU_HM_FAST(t, ip, |);
4901
4902         /* Thread. */
4903         thread_ip_inc(p);
4904 }
4905
4906 static inline void
4907 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
4908 {
4909         struct thread *t = &p->threads[p->thread_id];
4910         struct instruction *ip = t->ip;
4911
4912         TRACE("[Thread %2u] or (hh)\n", p->thread_id);
4913
4914         /* Structs. */
4915         ALU_HH_FAST(t, ip, |);
4916
4917         /* Thread. */
4918         thread_ip_inc(p);
4919 }
4920
4921 static inline void
4922 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4923 {
4924         struct thread *t = &p->threads[p->thread_id];
4925         struct instruction *ip = t->ip;
4926
4927         TRACE("[Thread %2u] or (i)\n", p->thread_id);
4928
4929         /* Structs. */
4930         ALU_I(t, ip, |);
4931
4932         /* Thread. */
4933         thread_ip_inc(p);
4934 }
4935
4936 static inline void
4937 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4938 {
4939         struct thread *t = &p->threads[p->thread_id];
4940         struct instruction *ip = t->ip;
4941
4942         TRACE("[Thread %2u] xor\n", p->thread_id);
4943
4944         /* Structs. */
4945         ALU(t, ip, ^);
4946
4947         /* Thread. */
4948         thread_ip_inc(p);
4949 }
4950
4951 static inline void
4952 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
4953 {
4954         struct thread *t = &p->threads[p->thread_id];
4955         struct instruction *ip = t->ip;
4956
4957         TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
4958
4959         /* Structs. */
4960         ALU_MH(t, ip, ^);
4961
4962         /* Thread. */
4963         thread_ip_inc(p);
4964 }
4965
4966 static inline void
4967 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
4968 {
4969         struct thread *t = &p->threads[p->thread_id];
4970         struct instruction *ip = t->ip;
4971
4972         TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
4973
4974         /* Structs. */
4975         ALU_HM_FAST(t, ip, ^);
4976
4977         /* Thread. */
4978         thread_ip_inc(p);
4979 }
4980
4981 static inline void
4982 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
4983 {
4984         struct thread *t = &p->threads[p->thread_id];
4985         struct instruction *ip = t->ip;
4986
4987         TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
4988
4989         /* Structs. */
4990         ALU_HH_FAST(t, ip, ^);
4991
4992         /* Thread. */
4993         thread_ip_inc(p);
4994 }
4995
4996 static inline void
4997 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4998 {
4999         struct thread *t = &p->threads[p->thread_id];
5000         struct instruction *ip = t->ip;
5001
5002         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
5003
5004         /* Structs. */
5005         ALU_I(t, ip, ^);
5006
5007         /* Thread. */
5008         thread_ip_inc(p);
5009 }
5010
5011 static inline void
5012 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
5013 {
5014         struct thread *t = &p->threads[p->thread_id];
5015         struct instruction *ip = t->ip;
5016         uint8_t *dst_struct, *src_struct;
5017         uint16_t *dst16_ptr, dst;
5018         uint64_t *src64_ptr, src64, src64_mask, src;
5019         uint64_t r;
5020
5021         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
5022
5023         /* Structs. */
5024         dst_struct = t->structs[ip->alu.dst.struct_id];
5025         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5026         dst = *dst16_ptr;
5027
5028         src_struct = t->structs[ip->alu.src.struct_id];
5029         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
5030         src64 = *src64_ptr;
5031         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
5032         src = src64 & src64_mask;
5033
5034         r = dst;
5035         r = ~r & 0xFFFF;
5036
5037         /* The first input (r) is a 16-bit number. The second and the third
5038          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
5039          * three numbers (output r) is a 34-bit number.
5040          */
5041         r += (src >> 32) + (src & 0xFFFFFFFF);
5042
5043         /* The first input is a 16-bit number. The second input is an 18-bit
5044          * number. In the worst case scenario, the sum of the two numbers is a
5045          * 19-bit number.
5046          */
5047         r = (r & 0xFFFF) + (r >> 16);
5048
5049         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5050          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
5051          */
5052         r = (r & 0xFFFF) + (r >> 16);
5053
5054         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5055          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5056          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
5057          * therefore the output r is always a 16-bit number.
5058          */
5059         r = (r & 0xFFFF) + (r >> 16);
5060
5061         r = ~r & 0xFFFF;
5062         r = r ? r : 0xFFFF;
5063
5064         *dst16_ptr = (uint16_t)r;
5065
5066         /* Thread. */
5067         thread_ip_inc(p);
5068 }
5069
5070 static inline void
5071 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
5072 {
5073         struct thread *t = &p->threads[p->thread_id];
5074         struct instruction *ip = t->ip;
5075         uint8_t *dst_struct, *src_struct;
5076         uint16_t *dst16_ptr, dst;
5077         uint64_t *src64_ptr, src64, src64_mask, src;
5078         uint64_t r;
5079
5080         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
5081
5082         /* Structs. */
5083         dst_struct = t->structs[ip->alu.dst.struct_id];
5084         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5085         dst = *dst16_ptr;
5086
5087         src_struct = t->structs[ip->alu.src.struct_id];
5088         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
5089         src64 = *src64_ptr;
5090         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
5091         src = src64 & src64_mask;
5092
5093         r = dst;
5094         r = ~r & 0xFFFF;
5095
5096         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
5097          * the following sequence of operations in 2's complement arithmetic:
5098          *    a '- b = (a - b) % 0xFFFF.
5099          *
5100          * In order to prevent an underflow for the below subtraction, in which
5101          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
5102          * minuend), we first add a multiple of the 0xFFFF modulus to the
5103          * minuend. The number we add to the minuend needs to be a 34-bit number
5104          * or higher, so for readability reasons we picked the 36-bit multiple.
5105          * We are effectively turning the 16-bit minuend into a 36-bit number:
5106          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
5107          */
5108         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
5109
5110         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
5111          * result (the output r) is a 36-bit number.
5112          */
5113         r -= (src >> 32) + (src & 0xFFFFFFFF);
5114
5115         /* The first input is a 16-bit number. The second input is a 20-bit
5116          * number. Their sum is a 21-bit number.
5117          */
5118         r = (r & 0xFFFF) + (r >> 16);
5119
5120         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5121          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
5122          */
5123         r = (r & 0xFFFF) + (r >> 16);
5124
5125         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5126          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5127          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5128          * generated, therefore the output r is always a 16-bit number.
5129          */
5130         r = (r & 0xFFFF) + (r >> 16);
5131
5132         r = ~r & 0xFFFF;
5133         r = r ? r : 0xFFFF;
5134
5135         *dst16_ptr = (uint16_t)r;
5136
5137         /* Thread. */
5138         thread_ip_inc(p);
5139 }
5140
5141 static inline void
5142 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
5143 {
5144         struct thread *t = &p->threads[p->thread_id];
5145         struct instruction *ip = t->ip;
5146         uint8_t *dst_struct, *src_struct;
5147         uint16_t *dst16_ptr;
5148         uint32_t *src32_ptr;
5149         uint64_t r0, r1;
5150
5151         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
5152
5153         /* Structs. */
5154         dst_struct = t->structs[ip->alu.dst.struct_id];
5155         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5156
5157         src_struct = t->structs[ip->alu.src.struct_id];
5158         src32_ptr = (uint32_t *)&src_struct[0];
5159
5160         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
5161         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
5162         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
5163         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
5164         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
5165
5166         /* The first input is a 16-bit number. The second input is a 19-bit
5167          * number. Their sum is a 20-bit number.
5168          */
5169         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5170
5171         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5172          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
5173          */
5174         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5175
5176         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5177          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5178          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
5179          * generated, therefore the output r is always a 16-bit number.
5180          */
5181         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5182
5183         r0 = ~r0 & 0xFFFF;
5184         r0 = r0 ? r0 : 0xFFFF;
5185
5186         *dst16_ptr = (uint16_t)r0;
5187
5188         /* Thread. */
5189         thread_ip_inc(p);
5190 }
5191
5192 static inline void
5193 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
5194 {
5195         struct thread *t = &p->threads[p->thread_id];
5196         struct instruction *ip = t->ip;
5197         uint8_t *dst_struct, *src_struct;
5198         uint16_t *dst16_ptr;
5199         uint32_t *src32_ptr;
5200         uint64_t r = 0;
5201         uint32_t i;
5202
5203         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
5204
5205         /* Structs. */
5206         dst_struct = t->structs[ip->alu.dst.struct_id];
5207         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5208
5209         src_struct = t->structs[ip->alu.src.struct_id];
5210         src32_ptr = (uint32_t *)&src_struct[0];
5211
5212         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
5213          * Therefore, in the worst case scenario, a 35-bit number is added to a
5214          * 16-bit number (the input r), so the output r is 36-bit number.
5215          */
5216         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
5217                 r += *src32_ptr;
5218
5219         /* The first input is a 16-bit number. The second input is a 20-bit
5220          * number. Their sum is a 21-bit number.
5221          */
5222         r = (r & 0xFFFF) + (r >> 16);
5223
5224         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5225          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
5226          */
5227         r = (r & 0xFFFF) + (r >> 16);
5228
5229         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5230          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5231          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5232          * generated, therefore the output r is always a 16-bit number.
5233          */
5234         r = (r & 0xFFFF) + (r >> 16);
5235
5236         r = ~r & 0xFFFF;
5237         r = r ? r : 0xFFFF;
5238
5239         *dst16_ptr = (uint16_t)r;
5240
5241         /* Thread. */
5242         thread_ip_inc(p);
5243 }
5244
5245 /*
5246  * Register array.
5247  */
5248 static struct regarray *
5249 regarray_find(struct rte_swx_pipeline *p, const char *name);
5250
5251 static int
5252 instr_regprefetch_translate(struct rte_swx_pipeline *p,
5253                       struct action *action,
5254                       char **tokens,
5255                       int n_tokens,
5256                       struct instruction *instr,
5257                       struct instruction_data *data __rte_unused)
5258 {
5259         char *regarray = tokens[1], *idx = tokens[2];
5260         struct regarray *r;
5261         struct field *fidx;
5262         uint32_t idx_struct_id, idx_val;
5263
5264         CHECK(n_tokens == 3, EINVAL);
5265
5266         r = regarray_find(p, regarray);
5267         CHECK(r, EINVAL);
5268
5269         /* REGPREFETCH_RH, REGPREFETCH_RM. */
5270         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5271         if (fidx) {
5272                 instr->type = INSTR_REGPREFETCH_RM;
5273                 if (idx[0] == 'h')
5274                         instr->type = INSTR_REGPREFETCH_RH;
5275
5276                 instr->regarray.regarray_id = r->id;
5277                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5278                 instr->regarray.idx.n_bits = fidx->n_bits;
5279                 instr->regarray.idx.offset = fidx->offset / 8;
5280                 instr->regarray.dstsrc_val = 0; /* Unused. */
5281                 return 0;
5282         }
5283
5284         /* REGPREFETCH_RI. */
5285         idx_val = strtoul(idx, &idx, 0);
5286         CHECK(!idx[0], EINVAL);
5287
5288         instr->type = INSTR_REGPREFETCH_RI;
5289         instr->regarray.regarray_id = r->id;
5290         instr->regarray.idx_val = idx_val;
5291         instr->regarray.dstsrc_val = 0; /* Unused. */
5292         return 0;
5293 }
5294
5295 static int
5296 instr_regrd_translate(struct rte_swx_pipeline *p,
5297                       struct action *action,
5298                       char **tokens,
5299                       int n_tokens,
5300                       struct instruction *instr,
5301                       struct instruction_data *data __rte_unused)
5302 {
5303         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
5304         struct regarray *r;
5305         struct field *fdst, *fidx;
5306         uint32_t dst_struct_id, idx_struct_id, idx_val;
5307
5308         CHECK(n_tokens == 4, EINVAL);
5309
5310         r = regarray_find(p, regarray);
5311         CHECK(r, EINVAL);
5312
5313         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
5314         CHECK(fdst, EINVAL);
5315
5316         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
5317         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5318         if (fidx) {
5319                 instr->type = INSTR_REGRD_MRM;
5320                 if (dst[0] == 'h' && idx[0] != 'h')
5321                         instr->type = INSTR_REGRD_HRM;
5322                 if (dst[0] != 'h' && idx[0] == 'h')
5323                         instr->type = INSTR_REGRD_MRH;
5324                 if (dst[0] == 'h' && idx[0] == 'h')
5325                         instr->type = INSTR_REGRD_HRH;
5326
5327                 instr->regarray.regarray_id = r->id;
5328                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5329                 instr->regarray.idx.n_bits = fidx->n_bits;
5330                 instr->regarray.idx.offset = fidx->offset / 8;
5331                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5332                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
5333                 instr->regarray.dstsrc.offset = fdst->offset / 8;
5334                 return 0;
5335         }
5336
5337         /* REGRD_MRI, REGRD_HRI. */
5338         idx_val = strtoul(idx, &idx, 0);
5339         CHECK(!idx[0], EINVAL);
5340
5341         instr->type = INSTR_REGRD_MRI;
5342         if (dst[0] == 'h')
5343                 instr->type = INSTR_REGRD_HRI;
5344
5345         instr->regarray.regarray_id = r->id;
5346         instr->regarray.idx_val = idx_val;
5347         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5348         instr->regarray.dstsrc.n_bits = fdst->n_bits;
5349         instr->regarray.dstsrc.offset = fdst->offset / 8;
5350         return 0;
5351 }
5352
5353 static int
5354 instr_regwr_translate(struct rte_swx_pipeline *p,
5355                       struct action *action,
5356                       char **tokens,
5357                       int n_tokens,
5358                       struct instruction *instr,
5359                       struct instruction_data *data __rte_unused)
5360 {
5361         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5362         struct regarray *r;
5363         struct field *fidx, *fsrc;
5364         uint64_t src_val;
5365         uint32_t idx_struct_id, idx_val, src_struct_id;
5366
5367         CHECK(n_tokens == 4, EINVAL);
5368
5369         r = regarray_find(p, regarray);
5370         CHECK(r, EINVAL);
5371
5372         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
5373         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5374         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5375         if (fidx && fsrc) {
5376                 instr->type = INSTR_REGWR_RMM;
5377                 if (idx[0] == 'h' && src[0] != 'h')
5378                         instr->type = INSTR_REGWR_RHM;
5379                 if (idx[0] != 'h' && src[0] == 'h')
5380                         instr->type = INSTR_REGWR_RMH;
5381                 if (idx[0] == 'h' && src[0] == 'h')
5382                         instr->type = INSTR_REGWR_RHH;
5383
5384                 instr->regarray.regarray_id = r->id;
5385                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5386                 instr->regarray.idx.n_bits = fidx->n_bits;
5387                 instr->regarray.idx.offset = fidx->offset / 8;
5388                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5389                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5390                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5391                 return 0;
5392         }
5393
5394         /* REGWR_RHI, REGWR_RMI. */
5395         if (fidx && !fsrc) {
5396                 src_val = strtoull(src, &src, 0);
5397                 CHECK(!src[0], EINVAL);
5398
5399                 instr->type = INSTR_REGWR_RMI;
5400                 if (idx[0] == 'h')
5401                         instr->type = INSTR_REGWR_RHI;
5402
5403                 instr->regarray.regarray_id = r->id;
5404                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5405                 instr->regarray.idx.n_bits = fidx->n_bits;
5406                 instr->regarray.idx.offset = fidx->offset / 8;
5407                 instr->regarray.dstsrc_val = src_val;
5408                 return 0;
5409         }
5410
5411         /* REGWR_RIH, REGWR_RIM. */
5412         if (!fidx && fsrc) {
5413                 idx_val = strtoul(idx, &idx, 0);
5414                 CHECK(!idx[0], EINVAL);
5415
5416                 instr->type = INSTR_REGWR_RIM;
5417                 if (src[0] == 'h')
5418                         instr->type = INSTR_REGWR_RIH;
5419
5420                 instr->regarray.regarray_id = r->id;
5421                 instr->regarray.idx_val = idx_val;
5422                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5423                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5424                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5425                 return 0;
5426         }
5427
5428         /* REGWR_RII. */
5429         src_val = strtoull(src, &src, 0);
5430         CHECK(!src[0], EINVAL);
5431
5432         idx_val = strtoul(idx, &idx, 0);
5433         CHECK(!idx[0], EINVAL);
5434
5435         instr->type = INSTR_REGWR_RII;
5436         instr->regarray.idx_val = idx_val;
5437         instr->regarray.dstsrc_val = src_val;
5438
5439         return 0;
5440 }
5441
5442 static int
5443 instr_regadd_translate(struct rte_swx_pipeline *p,
5444                        struct action *action,
5445                        char **tokens,
5446                        int n_tokens,
5447                        struct instruction *instr,
5448                        struct instruction_data *data __rte_unused)
5449 {
5450         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5451         struct regarray *r;
5452         struct field *fidx, *fsrc;
5453         uint64_t src_val;
5454         uint32_t idx_struct_id, idx_val, src_struct_id;
5455
5456         CHECK(n_tokens == 4, EINVAL);
5457
5458         r = regarray_find(p, regarray);
5459         CHECK(r, EINVAL);
5460
5461         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
5462         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5463         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5464         if (fidx && fsrc) {
5465                 instr->type = INSTR_REGADD_RMM;
5466                 if (idx[0] == 'h' && src[0] != 'h')
5467                         instr->type = INSTR_REGADD_RHM;
5468                 if (idx[0] != 'h' && src[0] == 'h')
5469                         instr->type = INSTR_REGADD_RMH;
5470                 if (idx[0] == 'h' && src[0] == 'h')
5471                         instr->type = INSTR_REGADD_RHH;
5472
5473                 instr->regarray.regarray_id = r->id;
5474                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5475                 instr->regarray.idx.n_bits = fidx->n_bits;
5476                 instr->regarray.idx.offset = fidx->offset / 8;
5477                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5478                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5479                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5480                 return 0;
5481         }
5482
5483         /* REGADD_RHI, REGADD_RMI. */
5484         if (fidx && !fsrc) {
5485                 src_val = strtoull(src, &src, 0);
5486                 CHECK(!src[0], EINVAL);
5487
5488                 instr->type = INSTR_REGADD_RMI;
5489                 if (idx[0] == 'h')
5490                         instr->type = INSTR_REGADD_RHI;
5491
5492                 instr->regarray.regarray_id = r->id;
5493                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5494                 instr->regarray.idx.n_bits = fidx->n_bits;
5495                 instr->regarray.idx.offset = fidx->offset / 8;
5496                 instr->regarray.dstsrc_val = src_val;
5497                 return 0;
5498         }
5499
5500         /* REGADD_RIH, REGADD_RIM. */
5501         if (!fidx && fsrc) {
5502                 idx_val = strtoul(idx, &idx, 0);
5503                 CHECK(!idx[0], EINVAL);
5504
5505                 instr->type = INSTR_REGADD_RIM;
5506                 if (src[0] == 'h')
5507                         instr->type = INSTR_REGADD_RIH;
5508
5509                 instr->regarray.regarray_id = r->id;
5510                 instr->regarray.idx_val = idx_val;
5511                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5512                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5513                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5514                 return 0;
5515         }
5516
5517         /* REGADD_RII. */
5518         src_val = strtoull(src, &src, 0);
5519         CHECK(!src[0], EINVAL);
5520
5521         idx_val = strtoul(idx, &idx, 0);
5522         CHECK(!idx[0], EINVAL);
5523
5524         instr->type = INSTR_REGADD_RII;
5525         instr->regarray.idx_val = idx_val;
5526         instr->regarray.dstsrc_val = src_val;
5527         return 0;
5528 }
5529
5530 static inline uint64_t *
5531 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
5532 {
5533         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5534         return r->regarray;
5535 }
5536
5537 static inline uint64_t
5538 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5539 {
5540         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5541
5542         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5543         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5544         uint64_t idx64 = *idx64_ptr;
5545         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
5546         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5547
5548         return idx;
5549 }
5550
5551 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5552
5553 static inline uint64_t
5554 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5555 {
5556         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5557
5558         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5559         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5560         uint64_t idx64 = *idx64_ptr;
5561         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
5562
5563         return idx;
5564 }
5565
5566 #else
5567
5568 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
5569
5570 #endif
5571
5572 static inline uint64_t
5573 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5574 {
5575         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5576
5577         uint64_t idx = ip->regarray.idx_val & r->size_mask;
5578
5579         return idx;
5580 }
5581
5582 static inline uint64_t
5583 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
5584 {
5585         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5586         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5587         uint64_t src64 = *src64_ptr;
5588         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5589         uint64_t src = src64 & src64_mask;
5590
5591         return src;
5592 }
5593
5594 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5595
5596 static inline uint64_t
5597 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
5598 {
5599         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5600         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5601         uint64_t src64 = *src64_ptr;
5602         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
5603
5604         return src;
5605 }
5606
5607 #else
5608
5609 #define instr_regarray_src_nbo instr_regarray_src_hbo
5610
5611 #endif
5612
5613 static inline void
5614 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5615 {
5616         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5617         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5618         uint64_t dst64 = *dst64_ptr;
5619         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5620
5621         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5622
5623 }
5624
5625 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5626
5627 static inline void
5628 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5629 {
5630         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5631         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5632         uint64_t dst64 = *dst64_ptr;
5633         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5634
5635         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
5636         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5637 }
5638
5639 #else
5640
5641 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
5642
5643 #endif
5644
5645 static inline void
5646 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
5647 {
5648         struct thread *t = &p->threads[p->thread_id];
5649         struct instruction *ip = t->ip;
5650         uint64_t *regarray, idx;
5651
5652         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
5653
5654         /* Structs. */
5655         regarray = instr_regarray_regarray(p, ip);
5656         idx = instr_regarray_idx_nbo(p, t, ip);
5657         rte_prefetch0(&regarray[idx]);
5658
5659         /* Thread. */
5660         thread_ip_inc(p);
5661 }
5662
5663 static inline void
5664 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
5665 {
5666         struct thread *t = &p->threads[p->thread_id];
5667         struct instruction *ip = t->ip;
5668         uint64_t *regarray, idx;
5669
5670         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
5671
5672         /* Structs. */
5673         regarray = instr_regarray_regarray(p, ip);
5674         idx = instr_regarray_idx_hbo(p, t, ip);
5675         rte_prefetch0(&regarray[idx]);
5676
5677         /* Thread. */
5678         thread_ip_inc(p);
5679 }
5680
5681 static inline void
5682 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
5683 {
5684         struct thread *t = &p->threads[p->thread_id];
5685         struct instruction *ip = t->ip;
5686         uint64_t *regarray, idx;
5687
5688         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
5689
5690         /* Structs. */
5691         regarray = instr_regarray_regarray(p, ip);
5692         idx = instr_regarray_idx_imm(p, ip);
5693         rte_prefetch0(&regarray[idx]);
5694
5695         /* Thread. */
5696         thread_ip_inc(p);
5697 }
5698
5699 static inline void
5700 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
5701 {
5702         struct thread *t = &p->threads[p->thread_id];
5703         struct instruction *ip = t->ip;
5704         uint64_t *regarray, idx;
5705
5706         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
5707
5708         /* Structs. */
5709         regarray = instr_regarray_regarray(p, ip);
5710         idx = instr_regarray_idx_nbo(p, t, ip);
5711         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5712
5713         /* Thread. */
5714         thread_ip_inc(p);
5715 }
5716
5717 static inline void
5718 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
5719 {
5720         struct thread *t = &p->threads[p->thread_id];
5721         struct instruction *ip = t->ip;
5722         uint64_t *regarray, idx;
5723
5724         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
5725
5726         /* Structs. */
5727         regarray = instr_regarray_regarray(p, ip);
5728         idx = instr_regarray_idx_hbo(p, t, ip);
5729         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5730
5731         /* Thread. */
5732         thread_ip_inc(p);
5733 }
5734
5735 static inline void
5736 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
5737 {
5738         struct thread *t = &p->threads[p->thread_id];
5739         struct instruction *ip = t->ip;
5740         uint64_t *regarray, idx;
5741
5742         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
5743
5744         /* Structs. */
5745         regarray = instr_regarray_regarray(p, ip);
5746         idx = instr_regarray_idx_nbo(p, t, ip);
5747         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5748
5749         /* Thread. */
5750         thread_ip_inc(p);
5751 }
5752
5753 static inline void
5754 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
5755 {
5756         struct thread *t = &p->threads[p->thread_id];
5757         struct instruction *ip = t->ip;
5758         uint64_t *regarray, idx;
5759
5760         /* Structs. */
5761         regarray = instr_regarray_regarray(p, ip);
5762         idx = instr_regarray_idx_hbo(p, t, ip);
5763         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5764
5765         /* Thread. */
5766         thread_ip_inc(p);
5767 }
5768
5769 static inline void
5770 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
5771 {
5772         struct thread *t = &p->threads[p->thread_id];
5773         struct instruction *ip = t->ip;
5774         uint64_t *regarray, idx;
5775
5776         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
5777
5778         /* Structs. */
5779         regarray = instr_regarray_regarray(p, ip);
5780         idx = instr_regarray_idx_imm(p, ip);
5781         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5782
5783         /* Thread. */
5784         thread_ip_inc(p);
5785 }
5786
5787 static inline void
5788 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
5789 {
5790         struct thread *t = &p->threads[p->thread_id];
5791         struct instruction *ip = t->ip;
5792         uint64_t *regarray, idx;
5793
5794         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
5795
5796         /* Structs. */
5797         regarray = instr_regarray_regarray(p, ip);
5798         idx = instr_regarray_idx_imm(p, ip);
5799         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5800
5801         /* Thread. */
5802         thread_ip_inc(p);
5803 }
5804
5805 static inline void
5806 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
5807 {
5808         struct thread *t = &p->threads[p->thread_id];
5809         struct instruction *ip = t->ip;
5810         uint64_t *regarray, idx, src;
5811
5812         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
5813
5814         /* Structs. */
5815         regarray = instr_regarray_regarray(p, ip);
5816         idx = instr_regarray_idx_nbo(p, t, ip);
5817         src = instr_regarray_src_nbo(t, ip);
5818         regarray[idx] = src;
5819
5820         /* Thread. */
5821         thread_ip_inc(p);
5822 }
5823
5824 static inline void
5825 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
5826 {
5827         struct thread *t = &p->threads[p->thread_id];
5828         struct instruction *ip = t->ip;
5829         uint64_t *regarray, idx, src;
5830
5831         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
5832
5833         /* Structs. */
5834         regarray = instr_regarray_regarray(p, ip);
5835         idx = instr_regarray_idx_nbo(p, t, ip);
5836         src = instr_regarray_src_hbo(t, ip);
5837         regarray[idx] = src;
5838
5839         /* Thread. */
5840         thread_ip_inc(p);
5841 }
5842
5843 static inline void
5844 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
5845 {
5846         struct thread *t = &p->threads[p->thread_id];
5847         struct instruction *ip = t->ip;
5848         uint64_t *regarray, idx, src;
5849
5850         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
5851
5852         /* Structs. */
5853         regarray = instr_regarray_regarray(p, ip);
5854         idx = instr_regarray_idx_hbo(p, t, ip);
5855         src = instr_regarray_src_nbo(t, ip);
5856         regarray[idx] = src;
5857
5858         /* Thread. */
5859         thread_ip_inc(p);
5860 }
5861
5862 static inline void
5863 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
5864 {
5865         struct thread *t = &p->threads[p->thread_id];
5866         struct instruction *ip = t->ip;
5867         uint64_t *regarray, idx, src;
5868
5869         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
5870
5871         /* Structs. */
5872         regarray = instr_regarray_regarray(p, ip);
5873         idx = instr_regarray_idx_hbo(p, t, ip);
5874         src = instr_regarray_src_hbo(t, ip);
5875         regarray[idx] = src;
5876
5877         /* Thread. */
5878         thread_ip_inc(p);
5879 }
5880
5881 static inline void
5882 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
5883 {
5884         struct thread *t = &p->threads[p->thread_id];
5885         struct instruction *ip = t->ip;
5886         uint64_t *regarray, idx, src;
5887
5888         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
5889
5890         /* Structs. */
5891         regarray = instr_regarray_regarray(p, ip);
5892         idx = instr_regarray_idx_nbo(p, t, ip);
5893         src = ip->regarray.dstsrc_val;
5894         regarray[idx] = src;
5895
5896         /* Thread. */
5897         thread_ip_inc(p);
5898 }
5899
5900 static inline void
5901 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
5902 {
5903         struct thread *t = &p->threads[p->thread_id];
5904         struct instruction *ip = t->ip;
5905         uint64_t *regarray, idx, src;
5906
5907         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
5908
5909         /* Structs. */
5910         regarray = instr_regarray_regarray(p, ip);
5911         idx = instr_regarray_idx_hbo(p, t, ip);
5912         src = ip->regarray.dstsrc_val;
5913         regarray[idx] = src;
5914
5915         /* Thread. */
5916         thread_ip_inc(p);
5917 }
5918
5919 static inline void
5920 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
5921 {
5922         struct thread *t = &p->threads[p->thread_id];
5923         struct instruction *ip = t->ip;
5924         uint64_t *regarray, idx, src;
5925
5926         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
5927
5928         /* Structs. */
5929         regarray = instr_regarray_regarray(p, ip);
5930         idx = instr_regarray_idx_imm(p, ip);
5931         src = instr_regarray_src_nbo(t, ip);
5932         regarray[idx] = src;
5933
5934         /* Thread. */
5935         thread_ip_inc(p);
5936 }
5937
5938 static inline void
5939 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
5940 {
5941         struct thread *t = &p->threads[p->thread_id];
5942         struct instruction *ip = t->ip;
5943         uint64_t *regarray, idx, src;
5944
5945         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
5946
5947         /* Structs. */
5948         regarray = instr_regarray_regarray(p, ip);
5949         idx = instr_regarray_idx_imm(p, ip);
5950         src = instr_regarray_src_hbo(t, ip);
5951         regarray[idx] = src;
5952
5953         /* Thread. */
5954         thread_ip_inc(p);
5955 }
5956
5957 static inline void
5958 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
5959 {
5960         struct thread *t = &p->threads[p->thread_id];
5961         struct instruction *ip = t->ip;
5962         uint64_t *regarray, idx, src;
5963
5964         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
5965
5966         /* Structs. */
5967         regarray = instr_regarray_regarray(p, ip);
5968         idx = instr_regarray_idx_imm(p, ip);
5969         src = ip->regarray.dstsrc_val;
5970         regarray[idx] = src;
5971
5972         /* Thread. */
5973         thread_ip_inc(p);
5974 }
5975
5976 static inline void
5977 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
5978 {
5979         struct thread *t = &p->threads[p->thread_id];
5980         struct instruction *ip = t->ip;
5981         uint64_t *regarray, idx, src;
5982
5983         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
5984
5985         /* Structs. */
5986         regarray = instr_regarray_regarray(p, ip);
5987         idx = instr_regarray_idx_nbo(p, t, ip);
5988         src = instr_regarray_src_nbo(t, ip);
5989         regarray[idx] += src;
5990
5991         /* Thread. */
5992         thread_ip_inc(p);
5993 }
5994
5995 static inline void
5996 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
5997 {
5998         struct thread *t = &p->threads[p->thread_id];
5999         struct instruction *ip = t->ip;
6000         uint64_t *regarray, idx, src;
6001
6002         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
6003
6004         /* Structs. */
6005         regarray = instr_regarray_regarray(p, ip);
6006         idx = instr_regarray_idx_nbo(p, t, ip);
6007         src = instr_regarray_src_hbo(t, ip);
6008         regarray[idx] += src;
6009
6010         /* Thread. */
6011         thread_ip_inc(p);
6012 }
6013
6014 static inline void
6015 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
6016 {
6017         struct thread *t = &p->threads[p->thread_id];
6018         struct instruction *ip = t->ip;
6019         uint64_t *regarray, idx, src;
6020
6021         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
6022
6023         /* Structs. */
6024         regarray = instr_regarray_regarray(p, ip);
6025         idx = instr_regarray_idx_hbo(p, t, ip);
6026         src = instr_regarray_src_nbo(t, ip);
6027         regarray[idx] += src;
6028
6029         /* Thread. */
6030         thread_ip_inc(p);
6031 }
6032
6033 static inline void
6034 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
6035 {
6036         struct thread *t = &p->threads[p->thread_id];
6037         struct instruction *ip = t->ip;
6038         uint64_t *regarray, idx, src;
6039
6040         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
6041
6042         /* Structs. */
6043         regarray = instr_regarray_regarray(p, ip);
6044         idx = instr_regarray_idx_hbo(p, t, ip);
6045         src = instr_regarray_src_hbo(t, ip);
6046         regarray[idx] += src;
6047
6048         /* Thread. */
6049         thread_ip_inc(p);
6050 }
6051
6052 static inline void
6053 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
6054 {
6055         struct thread *t = &p->threads[p->thread_id];
6056         struct instruction *ip = t->ip;
6057         uint64_t *regarray, idx, src;
6058
6059         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
6060
6061         /* Structs. */
6062         regarray = instr_regarray_regarray(p, ip);
6063         idx = instr_regarray_idx_nbo(p, t, ip);
6064         src = ip->regarray.dstsrc_val;
6065         regarray[idx] += src;
6066
6067         /* Thread. */
6068         thread_ip_inc(p);
6069 }
6070
6071 static inline void
6072 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
6073 {
6074         struct thread *t = &p->threads[p->thread_id];
6075         struct instruction *ip = t->ip;
6076         uint64_t *regarray, idx, src;
6077
6078         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
6079
6080         /* Structs. */
6081         regarray = instr_regarray_regarray(p, ip);
6082         idx = instr_regarray_idx_hbo(p, t, ip);
6083         src = ip->regarray.dstsrc_val;
6084         regarray[idx] += src;
6085
6086         /* Thread. */
6087         thread_ip_inc(p);
6088 }
6089
6090 static inline void
6091 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
6092 {
6093         struct thread *t = &p->threads[p->thread_id];
6094         struct instruction *ip = t->ip;
6095         uint64_t *regarray, idx, src;
6096
6097         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
6098
6099         /* Structs. */
6100         regarray = instr_regarray_regarray(p, ip);
6101         idx = instr_regarray_idx_imm(p, ip);
6102         src = instr_regarray_src_nbo(t, ip);
6103         regarray[idx] += src;
6104
6105         /* Thread. */
6106         thread_ip_inc(p);
6107 }
6108
6109 static inline void
6110 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
6111 {
6112         struct thread *t = &p->threads[p->thread_id];
6113         struct instruction *ip = t->ip;
6114         uint64_t *regarray, idx, src;
6115
6116         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
6117
6118         /* Structs. */
6119         regarray = instr_regarray_regarray(p, ip);
6120         idx = instr_regarray_idx_imm(p, ip);
6121         src = instr_regarray_src_hbo(t, ip);
6122         regarray[idx] += src;
6123
6124         /* Thread. */
6125         thread_ip_inc(p);
6126 }
6127
6128 static inline void
6129 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
6130 {
6131         struct thread *t = &p->threads[p->thread_id];
6132         struct instruction *ip = t->ip;
6133         uint64_t *regarray, idx, src;
6134
6135         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
6136
6137         /* Structs. */
6138         regarray = instr_regarray_regarray(p, ip);
6139         idx = instr_regarray_idx_imm(p, ip);
6140         src = ip->regarray.dstsrc_val;
6141         regarray[idx] += src;
6142
6143         /* Thread. */
6144         thread_ip_inc(p);
6145 }
6146
6147 /*
6148  * metarray.
6149  */
6150 static struct metarray *
6151 metarray_find(struct rte_swx_pipeline *p, const char *name);
6152
6153 static int
6154 instr_metprefetch_translate(struct rte_swx_pipeline *p,
6155                             struct action *action,
6156                             char **tokens,
6157                             int n_tokens,
6158                             struct instruction *instr,
6159                             struct instruction_data *data __rte_unused)
6160 {
6161         char *metarray = tokens[1], *idx = tokens[2];
6162         struct metarray *m;
6163         struct field *fidx;
6164         uint32_t idx_struct_id, idx_val;
6165
6166         CHECK(n_tokens == 3, EINVAL);
6167
6168         m = metarray_find(p, metarray);
6169         CHECK(m, EINVAL);
6170
6171         /* METPREFETCH_H, METPREFETCH_M. */
6172         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6173         if (fidx) {
6174                 instr->type = INSTR_METPREFETCH_M;
6175                 if (idx[0] == 'h')
6176                         instr->type = INSTR_METPREFETCH_H;
6177
6178                 instr->meter.metarray_id = m->id;
6179                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6180                 instr->meter.idx.n_bits = fidx->n_bits;
6181                 instr->meter.idx.offset = fidx->offset / 8;
6182                 return 0;
6183         }
6184
6185         /* METPREFETCH_I. */
6186         idx_val = strtoul(idx, &idx, 0);
6187         CHECK(!idx[0], EINVAL);
6188
6189         instr->type = INSTR_METPREFETCH_I;
6190         instr->meter.metarray_id = m->id;
6191         instr->meter.idx_val = idx_val;
6192         return 0;
6193 }
6194
6195 static int
6196 instr_meter_translate(struct rte_swx_pipeline *p,
6197                       struct action *action,
6198                       char **tokens,
6199                       int n_tokens,
6200                       struct instruction *instr,
6201                       struct instruction_data *data __rte_unused)
6202 {
6203         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
6204         char *color_in = tokens[4], *color_out = tokens[5];
6205         struct metarray *m;
6206         struct field *fidx, *flength, *fcin, *fcout;
6207         uint32_t idx_struct_id, length_struct_id;
6208         uint32_t color_in_struct_id, color_out_struct_id;
6209
6210         CHECK(n_tokens == 6, EINVAL);
6211
6212         m = metarray_find(p, metarray);
6213         CHECK(m, EINVAL);
6214
6215         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6216
6217         flength = struct_field_parse(p, action, length, &length_struct_id);
6218         CHECK(flength, EINVAL);
6219
6220         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
6221
6222         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
6223         CHECK(fcout, EINVAL);
6224
6225         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
6226         if (fidx && fcin) {
6227                 instr->type = INSTR_METER_MMM;
6228                 if (idx[0] == 'h' && length[0] == 'h')
6229                         instr->type = INSTR_METER_HHM;
6230                 if (idx[0] == 'h' && length[0] != 'h')
6231                         instr->type = INSTR_METER_HMM;
6232                 if (idx[0] != 'h' && length[0] == 'h')
6233                         instr->type = INSTR_METER_MHM;
6234
6235                 instr->meter.metarray_id = m->id;
6236
6237                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6238                 instr->meter.idx.n_bits = fidx->n_bits;
6239                 instr->meter.idx.offset = fidx->offset / 8;
6240
6241                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6242                 instr->meter.length.n_bits = flength->n_bits;
6243                 instr->meter.length.offset = flength->offset / 8;
6244
6245                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6246                 instr->meter.color_in.n_bits = fcin->n_bits;
6247                 instr->meter.color_in.offset = fcin->offset / 8;
6248
6249                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6250                 instr->meter.color_out.n_bits = fcout->n_bits;
6251                 instr->meter.color_out.offset = fcout->offset / 8;
6252
6253                 return 0;
6254         }
6255
6256         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
6257         if (fidx && !fcin) {
6258                 uint32_t color_in_val = strtoul(color_in, &color_in, 0);
6259                 CHECK(!color_in[0], EINVAL);
6260
6261                 instr->type = INSTR_METER_MMI;
6262                 if (idx[0] == 'h' && length[0] == 'h')
6263                         instr->type = INSTR_METER_HHI;
6264                 if (idx[0] == 'h' && length[0] != 'h')
6265                         instr->type = INSTR_METER_HMI;
6266                 if (idx[0] != 'h' && length[0] == 'h')
6267                         instr->type = INSTR_METER_MHI;
6268
6269                 instr->meter.metarray_id = m->id;
6270
6271                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6272                 instr->meter.idx.n_bits = fidx->n_bits;
6273                 instr->meter.idx.offset = fidx->offset / 8;
6274
6275                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6276                 instr->meter.length.n_bits = flength->n_bits;
6277                 instr->meter.length.offset = flength->offset / 8;
6278
6279                 instr->meter.color_in_val = color_in_val;
6280
6281                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6282                 instr->meter.color_out.n_bits = fcout->n_bits;
6283                 instr->meter.color_out.offset = fcout->offset / 8;
6284
6285                 return 0;
6286         }
6287
6288         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
6289         if (!fidx && fcin) {
6290                 uint32_t idx_val;
6291
6292                 idx_val = strtoul(idx, &idx, 0);
6293                 CHECK(!idx[0], EINVAL);
6294
6295                 instr->type = INSTR_METER_IMM;
6296                 if (length[0] == 'h')
6297                         instr->type = INSTR_METER_IHM;
6298
6299                 instr->meter.metarray_id = m->id;
6300
6301                 instr->meter.idx_val = idx_val;
6302
6303                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6304                 instr->meter.length.n_bits = flength->n_bits;
6305                 instr->meter.length.offset = flength->offset / 8;
6306
6307                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6308                 instr->meter.color_in.n_bits = fcin->n_bits;
6309                 instr->meter.color_in.offset = fcin->offset / 8;
6310
6311                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6312                 instr->meter.color_out.n_bits = fcout->n_bits;
6313                 instr->meter.color_out.offset = fcout->offset / 8;
6314
6315                 return 0;
6316         }
6317
6318         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
6319         if (!fidx && !fcin) {
6320                 uint32_t idx_val, color_in_val;
6321
6322                 idx_val = strtoul(idx, &idx, 0);
6323                 CHECK(!idx[0], EINVAL);
6324
6325                 color_in_val = strtoul(color_in, &color_in, 0);
6326                 CHECK(!color_in[0], EINVAL);
6327
6328                 instr->type = INSTR_METER_IMI;
6329                 if (length[0] == 'h')
6330                         instr->type = INSTR_METER_IHI;
6331
6332                 instr->meter.metarray_id = m->id;
6333
6334                 instr->meter.idx_val = idx_val;
6335
6336                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6337                 instr->meter.length.n_bits = flength->n_bits;
6338                 instr->meter.length.offset = flength->offset / 8;
6339
6340                 instr->meter.color_in_val = color_in_val;
6341
6342                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6343                 instr->meter.color_out.n_bits = fcout->n_bits;
6344                 instr->meter.color_out.offset = fcout->offset / 8;
6345
6346                 return 0;
6347         }
6348
6349         CHECK(0, EINVAL);
6350 }
6351
6352 static inline struct meter *
6353 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6354 {
6355         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6356
6357         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6358         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6359         uint64_t idx64 = *idx64_ptr;
6360         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
6361         uint64_t idx = idx64 & idx64_mask & r->size_mask;
6362
6363         return &r->metarray[idx];
6364 }
6365
6366 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6367
6368 static inline struct meter *
6369 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6370 {
6371         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6372
6373         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6374         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6375         uint64_t idx64 = *idx64_ptr;
6376         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
6377
6378         return &r->metarray[idx];
6379 }
6380
6381 #else
6382
6383 #define instr_meter_idx_nbo instr_meter_idx_hbo
6384
6385 #endif
6386
6387 static inline struct meter *
6388 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
6389 {
6390         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6391
6392         uint64_t idx =  ip->meter.idx_val & r->size_mask;
6393
6394         return &r->metarray[idx];
6395 }
6396
6397 static inline uint32_t
6398 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
6399 {
6400         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6401         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6402         uint64_t src64 = *src64_ptr;
6403         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
6404         uint64_t src = src64 & src64_mask;
6405
6406         return (uint32_t)src;
6407 }
6408
6409 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6410
6411 static inline uint32_t
6412 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
6413 {
6414         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6415         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6416         uint64_t src64 = *src64_ptr;
6417         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
6418
6419         return (uint32_t)src;
6420 }
6421
6422 #else
6423
6424 #define instr_meter_length_nbo instr_meter_length_hbo
6425
6426 #endif
6427
6428 static inline enum rte_color
6429 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
6430 {
6431         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
6432         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
6433         uint64_t src64 = *src64_ptr;
6434         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
6435         uint64_t src = src64 & src64_mask;
6436
6437         return (enum rte_color)src;
6438 }
6439
6440 static inline void
6441 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
6442 {
6443         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
6444         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
6445         uint64_t dst64 = *dst64_ptr;
6446         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
6447
6448         uint64_t src = (uint64_t)color_out;
6449
6450         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6451 }
6452
6453 static inline void
6454 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
6455 {
6456         struct thread *t = &p->threads[p->thread_id];
6457         struct instruction *ip = t->ip;
6458         struct meter *m;
6459
6460         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
6461
6462         /* Structs. */
6463         m = instr_meter_idx_nbo(p, t, ip);
6464         rte_prefetch0(m);
6465
6466         /* Thread. */
6467         thread_ip_inc(p);
6468 }
6469
6470 static inline void
6471 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
6472 {
6473         struct thread *t = &p->threads[p->thread_id];
6474         struct instruction *ip = t->ip;
6475         struct meter *m;
6476
6477         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
6478
6479         /* Structs. */
6480         m = instr_meter_idx_hbo(p, t, ip);
6481         rte_prefetch0(m);
6482
6483         /* Thread. */
6484         thread_ip_inc(p);
6485 }
6486
6487 static inline void
6488 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
6489 {
6490         struct thread *t = &p->threads[p->thread_id];
6491         struct instruction *ip = t->ip;
6492         struct meter *m;
6493
6494         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
6495
6496         /* Structs. */
6497         m = instr_meter_idx_imm(p, ip);
6498         rte_prefetch0(m);
6499
6500         /* Thread. */
6501         thread_ip_inc(p);
6502 }
6503
6504 static inline void
6505 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
6506 {
6507         struct thread *t = &p->threads[p->thread_id];
6508         struct instruction *ip = t->ip;
6509         struct meter *m;
6510         uint64_t time, n_pkts, n_bytes;
6511         uint32_t length;
6512         enum rte_color color_in, color_out;
6513
6514         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
6515
6516         /* Structs. */
6517         m = instr_meter_idx_nbo(p, t, ip);
6518         rte_prefetch0(m->n_pkts);
6519         time = rte_get_tsc_cycles();
6520         length = instr_meter_length_nbo(t, ip);
6521         color_in = instr_meter_color_in_hbo(t, ip);
6522
6523         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6524                 &m->profile->profile,
6525                 time,
6526                 length,
6527                 color_in);
6528
6529         color_out &= m->color_mask;
6530
6531         n_pkts = m->n_pkts[color_out];
6532         n_bytes = m->n_bytes[color_out];
6533
6534         instr_meter_color_out_hbo_set(t, ip, color_out);
6535
6536         m->n_pkts[color_out] = n_pkts + 1;
6537         m->n_bytes[color_out] = n_bytes + length;
6538
6539         /* Thread. */
6540         thread_ip_inc(p);
6541 }
6542
6543 static inline void
6544 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
6545 {
6546         struct thread *t = &p->threads[p->thread_id];
6547         struct instruction *ip = t->ip;
6548         struct meter *m;
6549         uint64_t time, n_pkts, n_bytes;
6550         uint32_t length;
6551         enum rte_color color_in, color_out;
6552
6553         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
6554
6555         /* Structs. */
6556         m = instr_meter_idx_nbo(p, t, ip);
6557         rte_prefetch0(m->n_pkts);
6558         time = rte_get_tsc_cycles();
6559         length = instr_meter_length_nbo(t, ip);
6560         color_in = (enum rte_color)ip->meter.color_in_val;
6561
6562         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6563                 &m->profile->profile,
6564                 time,
6565                 length,
6566                 color_in);
6567
6568         color_out &= m->color_mask;
6569
6570         n_pkts = m->n_pkts[color_out];
6571         n_bytes = m->n_bytes[color_out];
6572
6573         instr_meter_color_out_hbo_set(t, ip, color_out);
6574
6575         m->n_pkts[color_out] = n_pkts + 1;
6576         m->n_bytes[color_out] = n_bytes + length;
6577
6578         /* Thread. */
6579         thread_ip_inc(p);
6580 }
6581
6582 static inline void
6583 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
6584 {
6585         struct thread *t = &p->threads[p->thread_id];
6586         struct instruction *ip = t->ip;
6587         struct meter *m;
6588         uint64_t time, n_pkts, n_bytes;
6589         uint32_t length;
6590         enum rte_color color_in, color_out;
6591
6592         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
6593
6594         /* Structs. */
6595         m = instr_meter_idx_nbo(p, t, ip);
6596         rte_prefetch0(m->n_pkts);
6597         time = rte_get_tsc_cycles();
6598         length = instr_meter_length_hbo(t, ip);
6599         color_in = instr_meter_color_in_hbo(t, ip);
6600
6601         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6602                 &m->profile->profile,
6603                 time,
6604                 length,
6605                 color_in);
6606
6607         color_out &= m->color_mask;
6608
6609         n_pkts = m->n_pkts[color_out];
6610         n_bytes = m->n_bytes[color_out];
6611
6612         instr_meter_color_out_hbo_set(t, ip, color_out);
6613
6614         m->n_pkts[color_out] = n_pkts + 1;
6615         m->n_bytes[color_out] = n_bytes + length;
6616
6617         /* Thread. */
6618         thread_ip_inc(p);
6619 }
6620 static inline void
6621 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
6622 {
6623         struct thread *t = &p->threads[p->thread_id];
6624         struct instruction *ip = t->ip;
6625         struct meter *m;
6626         uint64_t time, n_pkts, n_bytes;
6627         uint32_t length;
6628         enum rte_color color_in, color_out;
6629
6630         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
6631
6632         /* Structs. */
6633         m = instr_meter_idx_nbo(p, t, ip);
6634         rte_prefetch0(m->n_pkts);
6635         time = rte_get_tsc_cycles();
6636         length = instr_meter_length_hbo(t, ip);
6637         color_in = (enum rte_color)ip->meter.color_in_val;
6638
6639         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6640                 &m->profile->profile,
6641                 time,
6642                 length,
6643                 color_in);
6644
6645         color_out &= m->color_mask;
6646
6647         n_pkts = m->n_pkts[color_out];
6648         n_bytes = m->n_bytes[color_out];
6649
6650         instr_meter_color_out_hbo_set(t, ip, color_out);
6651
6652         m->n_pkts[color_out] = n_pkts + 1;
6653         m->n_bytes[color_out] = n_bytes + length;
6654
6655         /* Thread. */
6656         thread_ip_inc(p);
6657 }
6658
6659 static inline void
6660 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
6661 {
6662         struct thread *t = &p->threads[p->thread_id];
6663         struct instruction *ip = t->ip;
6664         struct meter *m;
6665         uint64_t time, n_pkts, n_bytes;
6666         uint32_t length;
6667         enum rte_color color_in, color_out;
6668
6669         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
6670
6671         /* Structs. */
6672         m = instr_meter_idx_hbo(p, t, ip);
6673         rte_prefetch0(m->n_pkts);
6674         time = rte_get_tsc_cycles();
6675         length = instr_meter_length_nbo(t, ip);
6676         color_in = instr_meter_color_in_hbo(t, ip);
6677
6678         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6679                 &m->profile->profile,
6680                 time,
6681                 length,
6682                 color_in);
6683
6684         color_out &= m->color_mask;
6685
6686         n_pkts = m->n_pkts[color_out];
6687         n_bytes = m->n_bytes[color_out];
6688
6689         instr_meter_color_out_hbo_set(t, ip, color_out);
6690
6691         m->n_pkts[color_out] = n_pkts + 1;
6692         m->n_bytes[color_out] = n_bytes + length;
6693
6694         /* Thread. */
6695         thread_ip_inc(p);
6696 }
6697
6698 static inline void
6699 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
6700 {
6701         struct thread *t = &p->threads[p->thread_id];
6702         struct instruction *ip = t->ip;
6703         struct meter *m;
6704         uint64_t time, n_pkts, n_bytes;
6705         uint32_t length;
6706         enum rte_color color_in, color_out;
6707
6708         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
6709
6710         /* Structs. */
6711         m = instr_meter_idx_hbo(p, t, ip);
6712         rte_prefetch0(m->n_pkts);
6713         time = rte_get_tsc_cycles();
6714         length = instr_meter_length_nbo(t, ip);
6715         color_in = (enum rte_color)ip->meter.color_in_val;
6716
6717         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6718                 &m->profile->profile,
6719                 time,
6720                 length,
6721                 color_in);
6722
6723         color_out &= m->color_mask;
6724
6725         n_pkts = m->n_pkts[color_out];
6726         n_bytes = m->n_bytes[color_out];
6727
6728         instr_meter_color_out_hbo_set(t, ip, color_out);
6729
6730         m->n_pkts[color_out] = n_pkts + 1;
6731         m->n_bytes[color_out] = n_bytes + length;
6732
6733         /* Thread. */
6734         thread_ip_inc(p);
6735 }
6736
6737 static inline void
6738 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
6739 {
6740         struct thread *t = &p->threads[p->thread_id];
6741         struct instruction *ip = t->ip;
6742         struct meter *m;
6743         uint64_t time, n_pkts, n_bytes;
6744         uint32_t length;
6745         enum rte_color color_in, color_out;
6746
6747         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
6748
6749         /* Structs. */
6750         m = instr_meter_idx_hbo(p, t, ip);
6751         rte_prefetch0(m->n_pkts);
6752         time = rte_get_tsc_cycles();
6753         length = instr_meter_length_hbo(t, ip);
6754         color_in = instr_meter_color_in_hbo(t, ip);
6755
6756         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6757                 &m->profile->profile,
6758                 time,
6759                 length,
6760                 color_in);
6761
6762         color_out &= m->color_mask;
6763
6764         n_pkts = m->n_pkts[color_out];
6765         n_bytes = m->n_bytes[color_out];
6766
6767         instr_meter_color_out_hbo_set(t, ip, color_out);
6768
6769         m->n_pkts[color_out] = n_pkts + 1;
6770         m->n_bytes[color_out] = n_bytes + length;
6771
6772         /* Thread. */
6773         thread_ip_inc(p);
6774 }
6775
6776 static inline void
6777 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
6778 {
6779         struct thread *t = &p->threads[p->thread_id];
6780         struct instruction *ip = t->ip;
6781         struct meter *m;
6782         uint64_t time, n_pkts, n_bytes;
6783         uint32_t length;
6784         enum rte_color color_in, color_out;
6785
6786         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
6787
6788         /* Structs. */
6789         m = instr_meter_idx_hbo(p, t, ip);
6790         rte_prefetch0(m->n_pkts);
6791         time = rte_get_tsc_cycles();
6792         length = instr_meter_length_hbo(t, ip);
6793         color_in = (enum rte_color)ip->meter.color_in_val;
6794
6795         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6796                 &m->profile->profile,
6797                 time,
6798                 length,
6799                 color_in);
6800
6801         color_out &= m->color_mask;
6802
6803         n_pkts = m->n_pkts[color_out];
6804         n_bytes = m->n_bytes[color_out];
6805
6806         instr_meter_color_out_hbo_set(t, ip, color_out);
6807
6808         m->n_pkts[color_out] = n_pkts + 1;
6809         m->n_bytes[color_out] = n_bytes + length;
6810
6811         /* Thread. */
6812         thread_ip_inc(p);
6813 }
6814
6815 static inline void
6816 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
6817 {
6818         struct thread *t = &p->threads[p->thread_id];
6819         struct instruction *ip = t->ip;
6820         struct meter *m;
6821         uint64_t time, n_pkts, n_bytes;
6822         uint32_t length;
6823         enum rte_color color_in, color_out;
6824
6825         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
6826
6827         /* Structs. */
6828         m = instr_meter_idx_imm(p, ip);
6829         rte_prefetch0(m->n_pkts);
6830         time = rte_get_tsc_cycles();
6831         length = instr_meter_length_nbo(t, ip);
6832         color_in = instr_meter_color_in_hbo(t, ip);
6833
6834         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6835                 &m->profile->profile,
6836                 time,
6837                 length,
6838                 color_in);
6839
6840         color_out &= m->color_mask;
6841
6842         n_pkts = m->n_pkts[color_out];
6843         n_bytes = m->n_bytes[color_out];
6844
6845         instr_meter_color_out_hbo_set(t, ip, color_out);
6846
6847         m->n_pkts[color_out] = n_pkts + 1;
6848         m->n_bytes[color_out] = n_bytes + length;
6849
6850         /* Thread. */
6851         thread_ip_inc(p);
6852 }
6853
6854 static inline void
6855 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
6856 {
6857         struct thread *t = &p->threads[p->thread_id];
6858         struct instruction *ip = t->ip;
6859         struct meter *m;
6860         uint64_t time, n_pkts, n_bytes;
6861         uint32_t length;
6862         enum rte_color color_in, color_out;
6863
6864         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
6865
6866         /* Structs. */
6867         m = instr_meter_idx_imm(p, ip);
6868         rte_prefetch0(m->n_pkts);
6869         time = rte_get_tsc_cycles();
6870         length = instr_meter_length_nbo(t, ip);
6871         color_in = (enum rte_color)ip->meter.color_in_val;
6872
6873         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6874                 &m->profile->profile,
6875                 time,
6876                 length,
6877                 color_in);
6878
6879         color_out &= m->color_mask;
6880
6881         n_pkts = m->n_pkts[color_out];
6882         n_bytes = m->n_bytes[color_out];
6883
6884         instr_meter_color_out_hbo_set(t, ip, color_out);
6885
6886         m->n_pkts[color_out] = n_pkts + 1;
6887         m->n_bytes[color_out] = n_bytes + length;
6888
6889         /* Thread. */
6890         thread_ip_inc(p);
6891 }
6892
6893 static inline void
6894 instr_meter_imm_exec(struct rte_swx_pipeline *p)
6895 {
6896         struct thread *t = &p->threads[p->thread_id];
6897         struct instruction *ip = t->ip;
6898         struct meter *m;
6899         uint64_t time, n_pkts, n_bytes;
6900         uint32_t length;
6901         enum rte_color color_in, color_out;
6902
6903         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
6904
6905         /* Structs. */
6906         m = instr_meter_idx_imm(p, ip);
6907         rte_prefetch0(m->n_pkts);
6908         time = rte_get_tsc_cycles();
6909         length = instr_meter_length_hbo(t, ip);
6910         color_in = instr_meter_color_in_hbo(t, ip);
6911
6912         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6913                 &m->profile->profile,
6914                 time,
6915                 length,
6916                 color_in);
6917
6918         color_out &= m->color_mask;
6919
6920         n_pkts = m->n_pkts[color_out];
6921         n_bytes = m->n_bytes[color_out];
6922
6923         instr_meter_color_out_hbo_set(t, ip, color_out);
6924
6925         m->n_pkts[color_out] = n_pkts + 1;
6926         m->n_bytes[color_out] = n_bytes + length;
6927
6928         /* Thread. */
6929         thread_ip_inc(p);
6930 }
6931 static inline void
6932 instr_meter_imi_exec(struct rte_swx_pipeline *p)
6933 {
6934         struct thread *t = &p->threads[p->thread_id];
6935         struct instruction *ip = t->ip;
6936         struct meter *m;
6937         uint64_t time, n_pkts, n_bytes;
6938         uint32_t length;
6939         enum rte_color color_in, color_out;
6940
6941         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
6942
6943         /* Structs. */
6944         m = instr_meter_idx_imm(p, ip);
6945         rte_prefetch0(m->n_pkts);
6946         time = rte_get_tsc_cycles();
6947         length = instr_meter_length_hbo(t, ip);
6948         color_in = (enum rte_color)ip->meter.color_in_val;
6949
6950         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6951                 &m->profile->profile,
6952                 time,
6953                 length,
6954                 color_in);
6955
6956         color_out &= m->color_mask;
6957
6958         n_pkts = m->n_pkts[color_out];
6959         n_bytes = m->n_bytes[color_out];
6960
6961         instr_meter_color_out_hbo_set(t, ip, color_out);
6962
6963         m->n_pkts[color_out] = n_pkts + 1;
6964         m->n_bytes[color_out] = n_bytes + length;
6965
6966         /* Thread. */
6967         thread_ip_inc(p);
6968 }
6969
6970 /*
6971  * jmp.
6972  */
6973 static struct action *
6974 action_find(struct rte_swx_pipeline *p, const char *name);
6975
6976 static int
6977 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
6978                     struct action *action __rte_unused,
6979                     char **tokens,
6980                     int n_tokens,
6981                     struct instruction *instr,
6982                     struct instruction_data *data)
6983 {
6984         CHECK(n_tokens == 2, EINVAL);
6985
6986         strcpy(data->jmp_label, tokens[1]);
6987
6988         instr->type = INSTR_JMP;
6989         instr->jmp.ip = NULL; /* Resolved later. */
6990         return 0;
6991 }
6992
6993 static int
6994 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
6995                           struct action *action __rte_unused,
6996                           char **tokens,
6997                           int n_tokens,
6998                           struct instruction *instr,
6999                           struct instruction_data *data)
7000 {
7001         struct header *h;
7002
7003         CHECK(n_tokens == 3, EINVAL);
7004
7005         strcpy(data->jmp_label, tokens[1]);
7006
7007         h = header_parse(p, tokens[2]);
7008         CHECK(h, EINVAL);
7009
7010         instr->type = INSTR_JMP_VALID;
7011         instr->jmp.ip = NULL; /* Resolved later. */
7012         instr->jmp.header_id = h->id;
7013         return 0;
7014 }
7015
7016 static int
7017 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
7018                             struct action *action __rte_unused,
7019                             char **tokens,
7020                             int n_tokens,
7021                             struct instruction *instr,
7022                             struct instruction_data *data)
7023 {
7024         struct header *h;
7025
7026         CHECK(n_tokens == 3, EINVAL);
7027
7028         strcpy(data->jmp_label, tokens[1]);
7029
7030         h = header_parse(p, tokens[2]);
7031         CHECK(h, EINVAL);
7032
7033         instr->type = INSTR_JMP_INVALID;
7034         instr->jmp.ip = NULL; /* Resolved later. */
7035         instr->jmp.header_id = h->id;
7036         return 0;
7037 }
7038
7039 static int
7040 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
7041                         struct action *action,
7042                         char **tokens,
7043                         int n_tokens,
7044                         struct instruction *instr,
7045                         struct instruction_data *data)
7046 {
7047         CHECK(!action, EINVAL);
7048         CHECK(n_tokens == 2, EINVAL);
7049
7050         strcpy(data->jmp_label, tokens[1]);
7051
7052         instr->type = INSTR_JMP_HIT;
7053         instr->jmp.ip = NULL; /* Resolved later. */
7054         return 0;
7055 }
7056
7057 static int
7058 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
7059                          struct action *action,
7060                          char **tokens,
7061                          int n_tokens,
7062                          struct instruction *instr,
7063                          struct instruction_data *data)
7064 {
7065         CHECK(!action, EINVAL);
7066         CHECK(n_tokens == 2, EINVAL);
7067
7068         strcpy(data->jmp_label, tokens[1]);
7069
7070         instr->type = INSTR_JMP_MISS;
7071         instr->jmp.ip = NULL; /* Resolved later. */
7072         return 0;
7073 }
7074
7075 static int
7076 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
7077                                struct action *action,
7078                                char **tokens,
7079                                int n_tokens,
7080                                struct instruction *instr,
7081                                struct instruction_data *data)
7082 {
7083         struct action *a;
7084
7085         CHECK(!action, EINVAL);
7086         CHECK(n_tokens == 3, EINVAL);
7087
7088         strcpy(data->jmp_label, tokens[1]);
7089
7090         a = action_find(p, tokens[2]);
7091         CHECK(a, EINVAL);
7092
7093         instr->type = INSTR_JMP_ACTION_HIT;
7094         instr->jmp.ip = NULL; /* Resolved later. */
7095         instr->jmp.action_id = a->id;
7096         return 0;
7097 }
7098
7099 static int
7100 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
7101                                 struct action *action,
7102                                 char **tokens,
7103                                 int n_tokens,
7104                                 struct instruction *instr,
7105                                 struct instruction_data *data)
7106 {
7107         struct action *a;
7108
7109         CHECK(!action, EINVAL);
7110         CHECK(n_tokens == 3, EINVAL);
7111
7112         strcpy(data->jmp_label, tokens[1]);
7113
7114         a = action_find(p, tokens[2]);
7115         CHECK(a, EINVAL);
7116
7117         instr->type = INSTR_JMP_ACTION_MISS;
7118         instr->jmp.ip = NULL; /* Resolved later. */
7119         instr->jmp.action_id = a->id;
7120         return 0;
7121 }
7122
7123 static int
7124 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
7125                        struct action *action,
7126                        char **tokens,
7127                        int n_tokens,
7128                        struct instruction *instr,
7129                        struct instruction_data *data)
7130 {
7131         char *a = tokens[2], *b = tokens[3];
7132         struct field *fa, *fb;
7133         uint64_t b_val;
7134         uint32_t a_struct_id, b_struct_id;
7135
7136         CHECK(n_tokens == 4, EINVAL);
7137
7138         strcpy(data->jmp_label, tokens[1]);
7139
7140         fa = struct_field_parse(p, action, a, &a_struct_id);
7141         CHECK(fa, EINVAL);
7142
7143         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
7144         fb = struct_field_parse(p, action, b, &b_struct_id);
7145         if (fb) {
7146                 instr->type = INSTR_JMP_EQ;
7147                 if (a[0] != 'h' && b[0] == 'h')
7148                         instr->type = INSTR_JMP_EQ_MH;
7149                 if (a[0] == 'h' && b[0] != 'h')
7150                         instr->type = INSTR_JMP_EQ_HM;
7151                 if (a[0] == 'h' && b[0] == 'h')
7152                         instr->type = INSTR_JMP_EQ_HH;
7153                 instr->jmp.ip = NULL; /* Resolved later. */
7154
7155                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7156                 instr->jmp.a.n_bits = fa->n_bits;
7157                 instr->jmp.a.offset = fa->offset / 8;
7158                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7159                 instr->jmp.b.n_bits = fb->n_bits;
7160                 instr->jmp.b.offset = fb->offset / 8;
7161                 return 0;
7162         }
7163
7164         /* JMP_EQ_I. */
7165         b_val = strtoull(b, &b, 0);
7166         CHECK(!b[0], EINVAL);
7167
7168         if (a[0] == 'h')
7169                 b_val = hton64(b_val) >> (64 - fa->n_bits);
7170
7171         instr->type = INSTR_JMP_EQ_I;
7172         instr->jmp.ip = NULL; /* Resolved later. */
7173         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7174         instr->jmp.a.n_bits = fa->n_bits;
7175         instr->jmp.a.offset = fa->offset / 8;
7176         instr->jmp.b_val = b_val;
7177         return 0;
7178 }
7179
7180 static int
7181 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
7182                         struct action *action,
7183                         char **tokens,
7184                         int n_tokens,
7185                         struct instruction *instr,
7186                         struct instruction_data *data)
7187 {
7188         char *a = tokens[2], *b = tokens[3];
7189         struct field *fa, *fb;
7190         uint64_t b_val;
7191         uint32_t a_struct_id, b_struct_id;
7192
7193         CHECK(n_tokens == 4, EINVAL);
7194
7195         strcpy(data->jmp_label, tokens[1]);
7196
7197         fa = struct_field_parse(p, action, a, &a_struct_id);
7198         CHECK(fa, EINVAL);
7199
7200         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
7201         fb = struct_field_parse(p, action, b, &b_struct_id);
7202         if (fb) {
7203                 instr->type = INSTR_JMP_NEQ;
7204                 if (a[0] != 'h' && b[0] == 'h')
7205                         instr->type = INSTR_JMP_NEQ_MH;
7206                 if (a[0] == 'h' && b[0] != 'h')
7207                         instr->type = INSTR_JMP_NEQ_HM;
7208                 if (a[0] == 'h' && b[0] == 'h')
7209                         instr->type = INSTR_JMP_NEQ_HH;
7210                 instr->jmp.ip = NULL; /* Resolved later. */
7211
7212                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7213                 instr->jmp.a.n_bits = fa->n_bits;
7214                 instr->jmp.a.offset = fa->offset / 8;
7215                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7216                 instr->jmp.b.n_bits = fb->n_bits;
7217                 instr->jmp.b.offset = fb->offset / 8;
7218                 return 0;
7219         }
7220
7221         /* JMP_NEQ_I. */
7222         b_val = strtoull(b, &b, 0);
7223         CHECK(!b[0], EINVAL);
7224
7225         if (a[0] == 'h')
7226                 b_val = hton64(b_val) >> (64 - fa->n_bits);
7227
7228         instr->type = INSTR_JMP_NEQ_I;
7229         instr->jmp.ip = NULL; /* Resolved later. */
7230         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7231         instr->jmp.a.n_bits = fa->n_bits;
7232         instr->jmp.a.offset = fa->offset / 8;
7233         instr->jmp.b_val = b_val;
7234         return 0;
7235 }
7236
7237 static int
7238 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
7239                        struct action *action,
7240                        char **tokens,
7241                        int n_tokens,
7242                        struct instruction *instr,
7243                        struct instruction_data *data)
7244 {
7245         char *a = tokens[2], *b = tokens[3];
7246         struct field *fa, *fb;
7247         uint64_t b_val;
7248         uint32_t a_struct_id, b_struct_id;
7249
7250         CHECK(n_tokens == 4, EINVAL);
7251
7252         strcpy(data->jmp_label, tokens[1]);
7253
7254         fa = struct_field_parse(p, action, a, &a_struct_id);
7255         CHECK(fa, EINVAL);
7256
7257         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
7258         fb = struct_field_parse(p, action, b, &b_struct_id);
7259         if (fb) {
7260                 instr->type = INSTR_JMP_LT;
7261                 if (a[0] == 'h' && b[0] != 'h')
7262                         instr->type = INSTR_JMP_LT_HM;
7263                 if (a[0] != 'h' && b[0] == 'h')
7264                         instr->type = INSTR_JMP_LT_MH;
7265                 if (a[0] == 'h' && b[0] == 'h')
7266                         instr->type = INSTR_JMP_LT_HH;
7267                 instr->jmp.ip = NULL; /* Resolved later. */
7268
7269                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7270                 instr->jmp.a.n_bits = fa->n_bits;
7271                 instr->jmp.a.offset = fa->offset / 8;
7272                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7273                 instr->jmp.b.n_bits = fb->n_bits;
7274                 instr->jmp.b.offset = fb->offset / 8;
7275                 return 0;
7276         }
7277
7278         /* JMP_LT_MI, JMP_LT_HI. */
7279         b_val = strtoull(b, &b, 0);
7280         CHECK(!b[0], EINVAL);
7281
7282         instr->type = INSTR_JMP_LT_MI;
7283         if (a[0] == 'h')
7284                 instr->type = INSTR_JMP_LT_HI;
7285         instr->jmp.ip = NULL; /* Resolved later. */
7286
7287         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7288         instr->jmp.a.n_bits = fa->n_bits;
7289         instr->jmp.a.offset = fa->offset / 8;
7290         instr->jmp.b_val = b_val;
7291         return 0;
7292 }
7293
7294 static int
7295 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
7296                        struct action *action,
7297                        char **tokens,
7298                        int n_tokens,
7299                        struct instruction *instr,
7300                        struct instruction_data *data)
7301 {
7302         char *a = tokens[2], *b = tokens[3];
7303         struct field *fa, *fb;
7304         uint64_t b_val;
7305         uint32_t a_struct_id, b_struct_id;
7306
7307         CHECK(n_tokens == 4, EINVAL);
7308
7309         strcpy(data->jmp_label, tokens[1]);
7310
7311         fa = struct_field_parse(p, action, a, &a_struct_id);
7312         CHECK(fa, EINVAL);
7313
7314         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
7315         fb = struct_field_parse(p, action, b, &b_struct_id);
7316         if (fb) {
7317                 instr->type = INSTR_JMP_GT;
7318                 if (a[0] == 'h' && b[0] != 'h')
7319                         instr->type = INSTR_JMP_GT_HM;
7320                 if (a[0] != 'h' && b[0] == 'h')
7321                         instr->type = INSTR_JMP_GT_MH;
7322                 if (a[0] == 'h' && b[0] == 'h')
7323                         instr->type = INSTR_JMP_GT_HH;
7324                 instr->jmp.ip = NULL; /* Resolved later. */
7325
7326                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7327                 instr->jmp.a.n_bits = fa->n_bits;
7328                 instr->jmp.a.offset = fa->offset / 8;
7329                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7330                 instr->jmp.b.n_bits = fb->n_bits;
7331                 instr->jmp.b.offset = fb->offset / 8;
7332                 return 0;
7333         }
7334
7335         /* JMP_GT_MI, JMP_GT_HI. */
7336         b_val = strtoull(b, &b, 0);
7337         CHECK(!b[0], EINVAL);
7338
7339         instr->type = INSTR_JMP_GT_MI;
7340         if (a[0] == 'h')
7341                 instr->type = INSTR_JMP_GT_HI;
7342         instr->jmp.ip = NULL; /* Resolved later. */
7343
7344         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7345         instr->jmp.a.n_bits = fa->n_bits;
7346         instr->jmp.a.offset = fa->offset / 8;
7347         instr->jmp.b_val = b_val;
7348         return 0;
7349 }
7350
7351 static inline void
7352 instr_jmp_exec(struct rte_swx_pipeline *p)
7353 {
7354         struct thread *t = &p->threads[p->thread_id];
7355         struct instruction *ip = t->ip;
7356
7357         TRACE("[Thread %2u] jmp\n", p->thread_id);
7358
7359         thread_ip_set(t, ip->jmp.ip);
7360 }
7361
7362 static inline void
7363 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
7364 {
7365         struct thread *t = &p->threads[p->thread_id];
7366         struct instruction *ip = t->ip;
7367         uint32_t header_id = ip->jmp.header_id;
7368
7369         TRACE("[Thread %2u] jmpv\n", p->thread_id);
7370
7371         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
7372 }
7373
7374 static inline void
7375 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
7376 {
7377         struct thread *t = &p->threads[p->thread_id];
7378         struct instruction *ip = t->ip;
7379         uint32_t header_id = ip->jmp.header_id;
7380
7381         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
7382
7383         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
7384 }
7385
7386 static inline void
7387 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
7388 {
7389         struct thread *t = &p->threads[p->thread_id];
7390         struct instruction *ip = t->ip;
7391         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
7392
7393         TRACE("[Thread %2u] jmph\n", p->thread_id);
7394
7395         t->ip = ip_next[t->hit];
7396 }
7397
7398 static inline void
7399 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
7400 {
7401         struct thread *t = &p->threads[p->thread_id];
7402         struct instruction *ip = t->ip;
7403         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
7404
7405         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
7406
7407         t->ip = ip_next[t->hit];
7408 }
7409
7410 static inline void
7411 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
7412 {
7413         struct thread *t = &p->threads[p->thread_id];
7414         struct instruction *ip = t->ip;
7415
7416         TRACE("[Thread %2u] jmpa\n", p->thread_id);
7417
7418         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
7419 }
7420
7421 static inline void
7422 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
7423 {
7424         struct thread *t = &p->threads[p->thread_id];
7425         struct instruction *ip = t->ip;
7426
7427         TRACE("[Thread %2u] jmpna\n", p->thread_id);
7428
7429         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
7430 }
7431
7432 static inline void
7433 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
7434 {
7435         struct thread *t = &p->threads[p->thread_id];
7436         struct instruction *ip = t->ip;
7437
7438         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
7439
7440         JMP_CMP(t, ip, ==);
7441 }
7442
7443 static inline void
7444 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
7445 {
7446         struct thread *t = &p->threads[p->thread_id];
7447         struct instruction *ip = t->ip;
7448
7449         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
7450
7451         JMP_CMP_MH(t, ip, ==);
7452 }
7453
7454 static inline void
7455 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
7456 {
7457         struct thread *t = &p->threads[p->thread_id];
7458         struct instruction *ip = t->ip;
7459
7460         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
7461
7462         JMP_CMP_HM(t, ip, ==);
7463 }
7464
7465 static inline void
7466 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
7467 {
7468         struct thread *t = &p->threads[p->thread_id];
7469         struct instruction *ip = t->ip;
7470
7471         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
7472
7473         JMP_CMP_HH_FAST(t, ip, ==);
7474 }
7475
7476 static inline void
7477 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
7478 {
7479         struct thread *t = &p->threads[p->thread_id];
7480         struct instruction *ip = t->ip;
7481
7482         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
7483
7484         JMP_CMP_I(t, ip, ==);
7485 }
7486
7487 static inline void
7488 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
7489 {
7490         struct thread *t = &p->threads[p->thread_id];
7491         struct instruction *ip = t->ip;
7492
7493         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
7494
7495         JMP_CMP(t, ip, !=);
7496 }
7497
7498 static inline void
7499 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
7500 {
7501         struct thread *t = &p->threads[p->thread_id];
7502         struct instruction *ip = t->ip;
7503
7504         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
7505
7506         JMP_CMP_MH(t, ip, !=);
7507 }
7508
7509 static inline void
7510 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
7511 {
7512         struct thread *t = &p->threads[p->thread_id];
7513         struct instruction *ip = t->ip;
7514
7515         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
7516
7517         JMP_CMP_HM(t, ip, !=);
7518 }
7519
7520 static inline void
7521 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
7522 {
7523         struct thread *t = &p->threads[p->thread_id];
7524         struct instruction *ip = t->ip;
7525
7526         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
7527
7528         JMP_CMP_HH_FAST(t, ip, !=);
7529 }
7530
7531 static inline void
7532 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
7533 {
7534         struct thread *t = &p->threads[p->thread_id];
7535         struct instruction *ip = t->ip;
7536
7537         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
7538
7539         JMP_CMP_I(t, ip, !=);
7540 }
7541
7542 static inline void
7543 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
7544 {
7545         struct thread *t = &p->threads[p->thread_id];
7546         struct instruction *ip = t->ip;
7547
7548         TRACE("[Thread %2u] jmplt\n", p->thread_id);
7549
7550         JMP_CMP(t, ip, <);
7551 }
7552
7553 static inline void
7554 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
7555 {
7556         struct thread *t = &p->threads[p->thread_id];
7557         struct instruction *ip = t->ip;
7558
7559         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
7560
7561         JMP_CMP_MH(t, ip, <);
7562 }
7563
7564 static inline void
7565 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
7566 {
7567         struct thread *t = &p->threads[p->thread_id];
7568         struct instruction *ip = t->ip;
7569
7570         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
7571
7572         JMP_CMP_HM(t, ip, <);
7573 }
7574
7575 static inline void
7576 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
7577 {
7578         struct thread *t = &p->threads[p->thread_id];
7579         struct instruction *ip = t->ip;
7580
7581         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
7582
7583         JMP_CMP_HH(t, ip, <);
7584 }
7585
7586 static inline void
7587 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
7588 {
7589         struct thread *t = &p->threads[p->thread_id];
7590         struct instruction *ip = t->ip;
7591
7592         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
7593
7594         JMP_CMP_MI(t, ip, <);
7595 }
7596
7597 static inline void
7598 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
7599 {
7600         struct thread *t = &p->threads[p->thread_id];
7601         struct instruction *ip = t->ip;
7602
7603         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
7604
7605         JMP_CMP_HI(t, ip, <);
7606 }
7607
7608 static inline void
7609 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
7610 {
7611         struct thread *t = &p->threads[p->thread_id];
7612         struct instruction *ip = t->ip;
7613
7614         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
7615
7616         JMP_CMP(t, ip, >);
7617 }
7618
7619 static inline void
7620 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
7621 {
7622         struct thread *t = &p->threads[p->thread_id];
7623         struct instruction *ip = t->ip;
7624
7625         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
7626
7627         JMP_CMP_MH(t, ip, >);
7628 }
7629
7630 static inline void
7631 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
7632 {
7633         struct thread *t = &p->threads[p->thread_id];
7634         struct instruction *ip = t->ip;
7635
7636         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
7637
7638         JMP_CMP_HM(t, ip, >);
7639 }
7640
7641 static inline void
7642 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
7643 {
7644         struct thread *t = &p->threads[p->thread_id];
7645         struct instruction *ip = t->ip;
7646
7647         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
7648
7649         JMP_CMP_HH(t, ip, >);
7650 }
7651
7652 static inline void
7653 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
7654 {
7655         struct thread *t = &p->threads[p->thread_id];
7656         struct instruction *ip = t->ip;
7657
7658         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
7659
7660         JMP_CMP_MI(t, ip, >);
7661 }
7662
7663 static inline void
7664 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
7665 {
7666         struct thread *t = &p->threads[p->thread_id];
7667         struct instruction *ip = t->ip;
7668
7669         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
7670
7671         JMP_CMP_HI(t, ip, >);
7672 }
7673
7674 /*
7675  * return.
7676  */
7677 static int
7678 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
7679                        struct action *action,
7680                        char **tokens __rte_unused,
7681                        int n_tokens,
7682                        struct instruction *instr,
7683                        struct instruction_data *data __rte_unused)
7684 {
7685         CHECK(action, EINVAL);
7686         CHECK(n_tokens == 1, EINVAL);
7687
7688         instr->type = INSTR_RETURN;
7689         return 0;
7690 }
7691
7692 static inline void
7693 instr_return_exec(struct rte_swx_pipeline *p)
7694 {
7695         struct thread *t = &p->threads[p->thread_id];
7696
7697         TRACE("[Thread %2u] return\n", p->thread_id);
7698
7699         t->ip = t->ret;
7700 }
7701
7702 static int
7703 instr_translate(struct rte_swx_pipeline *p,
7704                 struct action *action,
7705                 char *string,
7706                 struct instruction *instr,
7707                 struct instruction_data *data)
7708 {
7709         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
7710         int n_tokens = 0, tpos = 0;
7711
7712         /* Parse the instruction string into tokens. */
7713         for ( ; ; ) {
7714                 char *token;
7715
7716                 token = strtok_r(string, " \t\v", &string);
7717                 if (!token)
7718                         break;
7719
7720                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
7721                 CHECK_NAME(token, EINVAL);
7722
7723                 tokens[n_tokens] = token;
7724                 n_tokens++;
7725         }
7726
7727         CHECK(n_tokens, EINVAL);
7728
7729         /* Handle the optional instruction label. */
7730         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
7731                 strcpy(data->label, tokens[0]);
7732
7733                 tpos += 2;
7734                 CHECK(n_tokens - tpos, EINVAL);
7735         }
7736
7737         /* Identify the instruction type. */
7738         if (!strcmp(tokens[tpos], "rx"))
7739                 return instr_rx_translate(p,
7740                                           action,
7741                                           &tokens[tpos],
7742                                           n_tokens - tpos,
7743                                           instr,
7744                                           data);
7745
7746         if (!strcmp(tokens[tpos], "tx"))
7747                 return instr_tx_translate(p,
7748                                           action,
7749                                           &tokens[tpos],
7750                                           n_tokens - tpos,
7751                                           instr,
7752                                           data);
7753
7754         if (!strcmp(tokens[tpos], "drop"))
7755                 return instr_drop_translate(p,
7756                                             action,
7757                                             &tokens[tpos],
7758                                             n_tokens - tpos,
7759                                             instr,
7760                                             data);
7761
7762         if (!strcmp(tokens[tpos], "extract"))
7763                 return instr_hdr_extract_translate(p,
7764                                                    action,
7765                                                    &tokens[tpos],
7766                                                    n_tokens - tpos,
7767                                                    instr,
7768                                                    data);
7769
7770         if (!strcmp(tokens[tpos], "emit"))
7771                 return instr_hdr_emit_translate(p,
7772                                                 action,
7773                                                 &tokens[tpos],
7774                                                 n_tokens - tpos,
7775                                                 instr,
7776                                                 data);
7777
7778         if (!strcmp(tokens[tpos], "validate"))
7779                 return instr_hdr_validate_translate(p,
7780                                                     action,
7781                                                     &tokens[tpos],
7782                                                     n_tokens - tpos,
7783                                                     instr,
7784                                                     data);
7785
7786         if (!strcmp(tokens[tpos], "invalidate"))
7787                 return instr_hdr_invalidate_translate(p,
7788                                                       action,
7789                                                       &tokens[tpos],
7790                                                       n_tokens - tpos,
7791                                                       instr,
7792                                                       data);
7793
7794         if (!strcmp(tokens[tpos], "mov"))
7795                 return instr_mov_translate(p,
7796                                            action,
7797                                            &tokens[tpos],
7798                                            n_tokens - tpos,
7799                                            instr,
7800                                            data);
7801
7802         if (!strcmp(tokens[tpos], "add"))
7803                 return instr_alu_add_translate(p,
7804                                                action,
7805                                                &tokens[tpos],
7806                                                n_tokens - tpos,
7807                                                instr,
7808                                                data);
7809
7810         if (!strcmp(tokens[tpos], "sub"))
7811                 return instr_alu_sub_translate(p,
7812                                                action,
7813                                                &tokens[tpos],
7814                                                n_tokens - tpos,
7815                                                instr,
7816                                                data);
7817
7818         if (!strcmp(tokens[tpos], "ckadd"))
7819                 return instr_alu_ckadd_translate(p,
7820                                                  action,
7821                                                  &tokens[tpos],
7822                                                  n_tokens - tpos,
7823                                                  instr,
7824                                                  data);
7825
7826         if (!strcmp(tokens[tpos], "cksub"))
7827                 return instr_alu_cksub_translate(p,
7828                                                  action,
7829                                                  &tokens[tpos],
7830                                                  n_tokens - tpos,
7831                                                  instr,
7832                                                  data);
7833
7834         if (!strcmp(tokens[tpos], "and"))
7835                 return instr_alu_and_translate(p,
7836                                                action,
7837                                                &tokens[tpos],
7838                                                n_tokens - tpos,
7839                                                instr,
7840                                                data);
7841
7842         if (!strcmp(tokens[tpos], "or"))
7843                 return instr_alu_or_translate(p,
7844                                               action,
7845                                               &tokens[tpos],
7846                                               n_tokens - tpos,
7847                                               instr,
7848                                               data);
7849
7850         if (!strcmp(tokens[tpos], "xor"))
7851                 return instr_alu_xor_translate(p,
7852                                                action,
7853                                                &tokens[tpos],
7854                                                n_tokens - tpos,
7855                                                instr,
7856                                                data);
7857
7858         if (!strcmp(tokens[tpos], "shl"))
7859                 return instr_alu_shl_translate(p,
7860                                                action,
7861                                                &tokens[tpos],
7862                                                n_tokens - tpos,
7863                                                instr,
7864                                                data);
7865
7866         if (!strcmp(tokens[tpos], "shr"))
7867                 return instr_alu_shr_translate(p,
7868                                                action,
7869                                                &tokens[tpos],
7870                                                n_tokens - tpos,
7871                                                instr,
7872                                                data);
7873
7874         if (!strcmp(tokens[tpos], "regprefetch"))
7875                 return instr_regprefetch_translate(p,
7876                                                    action,
7877                                                    &tokens[tpos],
7878                                                    n_tokens - tpos,
7879                                                    instr,
7880                                                    data);
7881
7882         if (!strcmp(tokens[tpos], "regrd"))
7883                 return instr_regrd_translate(p,
7884                                              action,
7885                                              &tokens[tpos],
7886                                              n_tokens - tpos,
7887                                              instr,
7888                                              data);
7889
7890         if (!strcmp(tokens[tpos], "regwr"))
7891                 return instr_regwr_translate(p,
7892                                              action,
7893                                              &tokens[tpos],
7894                                              n_tokens - tpos,
7895                                              instr,
7896                                              data);
7897
7898         if (!strcmp(tokens[tpos], "regadd"))
7899                 return instr_regadd_translate(p,
7900                                               action,
7901                                               &tokens[tpos],
7902                                               n_tokens - tpos,
7903                                               instr,
7904                                               data);
7905
7906         if (!strcmp(tokens[tpos], "metprefetch"))
7907                 return instr_metprefetch_translate(p,
7908                                                    action,
7909                                                    &tokens[tpos],
7910                                                    n_tokens - tpos,
7911                                                    instr,
7912                                                    data);
7913
7914         if (!strcmp(tokens[tpos], "meter"))
7915                 return instr_meter_translate(p,
7916                                              action,
7917                                              &tokens[tpos],
7918                                              n_tokens - tpos,
7919                                              instr,
7920                                              data);
7921
7922         if (!strcmp(tokens[tpos], "table"))
7923                 return instr_table_translate(p,
7924                                              action,
7925                                              &tokens[tpos],
7926                                              n_tokens - tpos,
7927                                              instr,
7928                                              data);
7929
7930         if (!strcmp(tokens[tpos], "extern"))
7931                 return instr_extern_translate(p,
7932                                               action,
7933                                               &tokens[tpos],
7934                                               n_tokens - tpos,
7935                                               instr,
7936                                               data);
7937
7938         if (!strcmp(tokens[tpos], "jmp"))
7939                 return instr_jmp_translate(p,
7940                                            action,
7941                                            &tokens[tpos],
7942                                            n_tokens - tpos,
7943                                            instr,
7944                                            data);
7945
7946         if (!strcmp(tokens[tpos], "jmpv"))
7947                 return instr_jmp_valid_translate(p,
7948                                                  action,
7949                                                  &tokens[tpos],
7950                                                  n_tokens - tpos,
7951                                                  instr,
7952                                                  data);
7953
7954         if (!strcmp(tokens[tpos], "jmpnv"))
7955                 return instr_jmp_invalid_translate(p,
7956                                                    action,
7957                                                    &tokens[tpos],
7958                                                    n_tokens - tpos,
7959                                                    instr,
7960                                                    data);
7961
7962         if (!strcmp(tokens[tpos], "jmph"))
7963                 return instr_jmp_hit_translate(p,
7964                                                action,
7965                                                &tokens[tpos],
7966                                                n_tokens - tpos,
7967                                                instr,
7968                                                data);
7969
7970         if (!strcmp(tokens[tpos], "jmpnh"))
7971                 return instr_jmp_miss_translate(p,
7972                                                 action,
7973                                                 &tokens[tpos],
7974                                                 n_tokens - tpos,
7975                                                 instr,
7976                                                 data);
7977
7978         if (!strcmp(tokens[tpos], "jmpa"))
7979                 return instr_jmp_action_hit_translate(p,
7980                                                       action,
7981                                                       &tokens[tpos],
7982                                                       n_tokens - tpos,
7983                                                       instr,
7984                                                       data);
7985
7986         if (!strcmp(tokens[tpos], "jmpna"))
7987                 return instr_jmp_action_miss_translate(p,
7988                                                        action,
7989                                                        &tokens[tpos],
7990                                                        n_tokens - tpos,
7991                                                        instr,
7992                                                        data);
7993
7994         if (!strcmp(tokens[tpos], "jmpeq"))
7995                 return instr_jmp_eq_translate(p,
7996                                               action,
7997                                               &tokens[tpos],
7998                                               n_tokens - tpos,
7999                                               instr,
8000                                               data);
8001
8002         if (!strcmp(tokens[tpos], "jmpneq"))
8003                 return instr_jmp_neq_translate(p,
8004                                                action,
8005                                                &tokens[tpos],
8006                                                n_tokens - tpos,
8007                                                instr,
8008                                                data);
8009
8010         if (!strcmp(tokens[tpos], "jmplt"))
8011                 return instr_jmp_lt_translate(p,
8012                                               action,
8013                                               &tokens[tpos],
8014                                               n_tokens - tpos,
8015                                               instr,
8016                                               data);
8017
8018         if (!strcmp(tokens[tpos], "jmpgt"))
8019                 return instr_jmp_gt_translate(p,
8020                                               action,
8021                                               &tokens[tpos],
8022                                               n_tokens - tpos,
8023                                               instr,
8024                                               data);
8025
8026         if (!strcmp(tokens[tpos], "return"))
8027                 return instr_return_translate(p,
8028                                               action,
8029                                               &tokens[tpos],
8030                                               n_tokens - tpos,
8031                                               instr,
8032                                               data);
8033
8034         CHECK(0, EINVAL);
8035 }
8036
8037 static struct instruction_data *
8038 label_find(struct instruction_data *data, uint32_t n, const char *label)
8039 {
8040         uint32_t i;
8041
8042         for (i = 0; i < n; i++)
8043                 if (!strcmp(label, data[i].label))
8044                         return &data[i];
8045
8046         return NULL;
8047 }
8048
8049 static uint32_t
8050 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
8051 {
8052         uint32_t count = 0, i;
8053
8054         if (!label[0])
8055                 return 0;
8056
8057         for (i = 0; i < n; i++)
8058                 if (!strcmp(label, data[i].jmp_label))
8059                         count++;
8060
8061         return count;
8062 }
8063
8064 static int
8065 instr_label_check(struct instruction_data *instruction_data,
8066                   uint32_t n_instructions)
8067 {
8068         uint32_t i;
8069
8070         /* Check that all instruction labels are unique. */
8071         for (i = 0; i < n_instructions; i++) {
8072                 struct instruction_data *data = &instruction_data[i];
8073                 char *label = data->label;
8074                 uint32_t j;
8075
8076                 if (!label[0])
8077                         continue;
8078
8079                 for (j = i + 1; j < n_instructions; j++)
8080                         CHECK(strcmp(label, data[j].label), EINVAL);
8081         }
8082
8083         /* Get users for each instruction label. */
8084         for (i = 0; i < n_instructions; i++) {
8085                 struct instruction_data *data = &instruction_data[i];
8086                 char *label = data->label;
8087
8088                 data->n_users = label_is_used(instruction_data,
8089                                               n_instructions,
8090                                               label);
8091         }
8092
8093         return 0;
8094 }
8095
8096 static int
8097 instr_jmp_resolve(struct instruction *instructions,
8098                   struct instruction_data *instruction_data,
8099                   uint32_t n_instructions)
8100 {
8101         uint32_t i;
8102
8103         for (i = 0; i < n_instructions; i++) {
8104                 struct instruction *instr = &instructions[i];
8105                 struct instruction_data *data = &instruction_data[i];
8106                 struct instruction_data *found;
8107
8108                 if (!instruction_is_jmp(instr))
8109                         continue;
8110
8111                 found = label_find(instruction_data,
8112                                    n_instructions,
8113                                    data->jmp_label);
8114                 CHECK(found, EINVAL);
8115
8116                 instr->jmp.ip = &instructions[found - instruction_data];
8117         }
8118
8119         return 0;
8120 }
8121
8122 static int
8123 instr_verify(struct rte_swx_pipeline *p __rte_unused,
8124              struct action *a,
8125              struct instruction *instr,
8126              struct instruction_data *data __rte_unused,
8127              uint32_t n_instructions)
8128 {
8129         if (!a) {
8130                 enum instruction_type type;
8131                 uint32_t i;
8132
8133                 /* Check that the first instruction is rx. */
8134                 CHECK(instr[0].type == INSTR_RX, EINVAL);
8135
8136                 /* Check that there is at least one tx instruction. */
8137                 for (i = 0; i < n_instructions; i++) {
8138                         type = instr[i].type;
8139
8140                         if (instruction_is_tx(type))
8141                                 break;
8142                 }
8143                 CHECK(i < n_instructions, EINVAL);
8144
8145                 /* Check that the last instruction is either tx or unconditional
8146                  * jump.
8147                  */
8148                 type = instr[n_instructions - 1].type;
8149                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
8150         }
8151
8152         if (a) {
8153                 enum instruction_type type;
8154                 uint32_t i;
8155
8156                 /* Check that there is at least one return or tx instruction. */
8157                 for (i = 0; i < n_instructions; i++) {
8158                         type = instr[i].type;
8159
8160                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
8161                                 break;
8162                 }
8163                 CHECK(i < n_instructions, EINVAL);
8164         }
8165
8166         return 0;
8167 }
8168
8169 static uint32_t
8170 instr_compact(struct instruction *instructions,
8171               struct instruction_data *instruction_data,
8172               uint32_t n_instructions)
8173 {
8174         uint32_t i, pos = 0;
8175
8176         /* Eliminate the invalid instructions that have been optimized out. */
8177         for (i = 0; i < n_instructions; i++) {
8178                 struct instruction *instr = &instructions[i];
8179                 struct instruction_data *data = &instruction_data[i];
8180
8181                 if (data->invalid)
8182                         continue;
8183
8184                 if (i != pos) {
8185                         memcpy(&instructions[pos], instr, sizeof(*instr));
8186                         memcpy(&instruction_data[pos], data, sizeof(*data));
8187                 }
8188
8189                 pos++;
8190         }
8191
8192         return pos;
8193 }
8194
8195 static int
8196 instr_pattern_extract_many_search(struct instruction *instr,
8197                                   struct instruction_data *data,
8198                                   uint32_t n_instr,
8199                                   uint32_t *n_pattern_instr)
8200 {
8201         uint32_t i;
8202
8203         for (i = 0; i < n_instr; i++) {
8204                 if (data[i].invalid)
8205                         break;
8206
8207                 if (instr[i].type != INSTR_HDR_EXTRACT)
8208                         break;
8209
8210                 if (i == RTE_DIM(instr->io.hdr.header_id))
8211                         break;
8212
8213                 if (i && data[i].n_users)
8214                         break;
8215         }
8216
8217         if (i < 2)
8218                 return 0;
8219
8220         *n_pattern_instr = i;
8221         return 1;
8222 }
8223
8224 static void
8225 instr_pattern_extract_many_replace(struct instruction *instr,
8226                                    struct instruction_data *data,
8227                                    uint32_t n_instr)
8228 {
8229         uint32_t i;
8230
8231         for (i = 1; i < n_instr; i++) {
8232                 instr[0].type++;
8233                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8234                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8235                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8236
8237                 data[i].invalid = 1;
8238         }
8239 }
8240
8241 static uint32_t
8242 instr_pattern_extract_many_optimize(struct instruction *instructions,
8243                                     struct instruction_data *instruction_data,
8244                                     uint32_t n_instructions)
8245 {
8246         uint32_t i;
8247
8248         for (i = 0; i < n_instructions; ) {
8249                 struct instruction *instr = &instructions[i];
8250                 struct instruction_data *data = &instruction_data[i];
8251                 uint32_t n_instr = 0;
8252                 int detected;
8253
8254                 /* Extract many. */
8255                 detected = instr_pattern_extract_many_search(instr,
8256                                                              data,
8257                                                              n_instructions - i,
8258                                                              &n_instr);
8259                 if (detected) {
8260                         instr_pattern_extract_many_replace(instr,
8261                                                            data,
8262                                                            n_instr);
8263                         i += n_instr;
8264                         continue;
8265                 }
8266
8267                 /* No pattern starting at the current instruction. */
8268                 i++;
8269         }
8270
8271         /* Eliminate the invalid instructions that have been optimized out. */
8272         n_instructions = instr_compact(instructions,
8273                                        instruction_data,
8274                                        n_instructions);
8275
8276         return n_instructions;
8277 }
8278
8279 static int
8280 instr_pattern_emit_many_tx_search(struct instruction *instr,
8281                                   struct instruction_data *data,
8282                                   uint32_t n_instr,
8283                                   uint32_t *n_pattern_instr)
8284 {
8285         uint32_t i;
8286
8287         for (i = 0; i < n_instr; i++) {
8288                 if (data[i].invalid)
8289                         break;
8290
8291                 if (instr[i].type != INSTR_HDR_EMIT)
8292                         break;
8293
8294                 if (i == RTE_DIM(instr->io.hdr.header_id))
8295                         break;
8296
8297                 if (i && data[i].n_users)
8298                         break;
8299         }
8300
8301         if (!i)
8302                 return 0;
8303
8304         if (!instruction_is_tx(instr[i].type))
8305                 return 0;
8306
8307         if (data[i].n_users)
8308                 return 0;
8309
8310         i++;
8311
8312         *n_pattern_instr = i;
8313         return 1;
8314 }
8315
8316 static void
8317 instr_pattern_emit_many_tx_replace(struct instruction *instr,
8318                                    struct instruction_data *data,
8319                                    uint32_t n_instr)
8320 {
8321         uint32_t i;
8322
8323         /* Any emit instruction in addition to the first one. */
8324         for (i = 1; i < n_instr - 1; i++) {
8325                 instr[0].type++;
8326                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8327                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8328                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8329
8330                 data[i].invalid = 1;
8331         }
8332
8333         /* The TX instruction is the last one in the pattern. */
8334         instr[0].type++;
8335         instr[0].io.io.offset = instr[i].io.io.offset;
8336         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
8337         data[i].invalid = 1;
8338 }
8339
8340 static uint32_t
8341 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
8342                                     struct instruction_data *instruction_data,
8343                                     uint32_t n_instructions)
8344 {
8345         uint32_t i;
8346
8347         for (i = 0; i < n_instructions; ) {
8348                 struct instruction *instr = &instructions[i];
8349                 struct instruction_data *data = &instruction_data[i];
8350                 uint32_t n_instr = 0;
8351                 int detected;
8352
8353                 /* Emit many + TX. */
8354                 detected = instr_pattern_emit_many_tx_search(instr,
8355                                                              data,
8356                                                              n_instructions - i,
8357                                                              &n_instr);
8358                 if (detected) {
8359                         instr_pattern_emit_many_tx_replace(instr,
8360                                                            data,
8361                                                            n_instr);
8362                         i += n_instr;
8363                         continue;
8364                 }
8365
8366                 /* No pattern starting at the current instruction. */
8367                 i++;
8368         }
8369
8370         /* Eliminate the invalid instructions that have been optimized out. */
8371         n_instructions = instr_compact(instructions,
8372                                        instruction_data,
8373                                        n_instructions);
8374
8375         return n_instructions;
8376 }
8377
8378 static uint32_t
8379 action_arg_src_mov_count(struct action *a,
8380                          uint32_t arg_id,
8381                          struct instruction *instructions,
8382                          struct instruction_data *instruction_data,
8383                          uint32_t n_instructions);
8384
8385 static int
8386 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
8387                                       struct action *a,
8388                                       struct instruction *instr,
8389                                       struct instruction_data *data,
8390                                       uint32_t n_instr,
8391                                       struct instruction *instructions,
8392                                       struct instruction_data *instruction_data,
8393                                       uint32_t n_instructions,
8394                                       uint32_t *n_pattern_instr)
8395 {
8396         struct header *h;
8397         uint32_t src_field_id, i, j;
8398
8399         /* Prerequisites. */
8400         if (!a || !a->st)
8401                 return 0;
8402
8403         /* First instruction: MOV_HM. */
8404         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
8405                 return 0;
8406
8407         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
8408         if (!h)
8409                 return 0;
8410
8411         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
8412                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
8413                         break;
8414
8415         if (src_field_id == a->st->n_fields)
8416                 return 0;
8417
8418         if (instr[0].mov.dst.offset ||
8419             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
8420             instr[0].mov.src.struct_id ||
8421             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
8422             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
8423                 return 0;
8424
8425         if ((n_instr < h->st->n_fields + 1) ||
8426              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
8427                 return 0;
8428
8429         /* Subsequent instructions: MOV_HM. */
8430         for (i = 1; i < h->st->n_fields; i++)
8431                 if (data[i].invalid ||
8432                     data[i].n_users ||
8433                     (instr[i].type != INSTR_MOV_HM) ||
8434                     (instr[i].mov.dst.struct_id != h->struct_id) ||
8435                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
8436                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
8437                     instr[i].mov.src.struct_id ||
8438                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
8439                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
8440                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
8441                         return 0;
8442
8443         /* Last instruction: HDR_VALIDATE. */
8444         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
8445             (instr[i].valid.header_id != h->id))
8446                 return 0;
8447
8448         /* Check that none of the action args that are used as source for this
8449          * DMA transfer are not used as source in any other mov instruction.
8450          */
8451         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
8452                 uint32_t n_users;
8453
8454                 n_users = action_arg_src_mov_count(a,
8455                                                    j,
8456                                                    instructions,
8457                                                    instruction_data,
8458                                                    n_instructions);
8459                 if (n_users > 1)
8460                         return 0;
8461         }
8462
8463         *n_pattern_instr = 1 + i;
8464         return 1;
8465 }
8466
8467 static void
8468 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
8469                                        struct action *a,
8470                                        struct instruction *instr,
8471                                        struct instruction_data *data,
8472                                        uint32_t n_instr)
8473 {
8474         struct header *h;
8475         uint32_t src_field_id, src_offset, i;
8476
8477         /* Read from the instructions before they are modified. */
8478         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
8479         if (!h)
8480                 return;
8481
8482         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
8483                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
8484                         break;
8485
8486         if (src_field_id == a->st->n_fields)
8487                 return;
8488
8489         src_offset = instr[0].mov.src.offset;
8490
8491         /* Modify the instructions. */
8492         instr[0].type = INSTR_DMA_HT;
8493         instr[0].dma.dst.header_id[0] = h->id;
8494         instr[0].dma.dst.struct_id[0] = h->struct_id;
8495         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
8496         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
8497
8498         for (i = 1; i < n_instr; i++)
8499                 data[i].invalid = 1;
8500
8501         /* Update the endianness of the action arguments to header endianness. */
8502         for (i = 0; i < h->st->n_fields; i++)
8503                 a->args_endianness[src_field_id + i] = 1;
8504 }
8505
8506 static uint32_t
8507 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
8508                                         struct action *a,
8509                                         struct instruction *instructions,
8510                                         struct instruction_data *instruction_data,
8511                                         uint32_t n_instructions)
8512 {
8513         uint32_t i;
8514
8515         if (!a || !a->st)
8516                 return n_instructions;
8517
8518         for (i = 0; i < n_instructions; ) {
8519                 struct instruction *instr = &instructions[i];
8520                 struct instruction_data *data = &instruction_data[i];
8521                 uint32_t n_instr = 0;
8522                 int detected;
8523
8524                 /* Mov all + validate. */
8525                 detected = instr_pattern_mov_all_validate_search(p,
8526                                                                  a,
8527                                                                  instr,
8528                                                                  data,
8529                                                                  n_instructions - i,
8530                                                                  instructions,
8531                                                                  instruction_data,
8532                                                                  n_instructions,
8533                                                                  &n_instr);
8534                 if (detected) {
8535                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
8536                         i += n_instr;
8537                         continue;
8538                 }
8539
8540                 /* No pattern starting at the current instruction. */
8541                 i++;
8542         }
8543
8544         /* Eliminate the invalid instructions that have been optimized out. */
8545         n_instructions = instr_compact(instructions,
8546                                        instruction_data,
8547                                        n_instructions);
8548
8549         return n_instructions;
8550 }
8551
8552 static int
8553 instr_pattern_dma_many_search(struct instruction *instr,
8554                               struct instruction_data *data,
8555                               uint32_t n_instr,
8556                               uint32_t *n_pattern_instr)
8557 {
8558         uint32_t i;
8559
8560         for (i = 0; i < n_instr; i++) {
8561                 if (data[i].invalid)
8562                         break;
8563
8564                 if (instr[i].type != INSTR_DMA_HT)
8565                         break;
8566
8567                 if (i == RTE_DIM(instr->dma.dst.header_id))
8568                         break;
8569
8570                 if (i && data[i].n_users)
8571                         break;
8572         }
8573
8574         if (i < 2)
8575                 return 0;
8576
8577         *n_pattern_instr = i;
8578         return 1;
8579 }
8580
8581 static void
8582 instr_pattern_dma_many_replace(struct instruction *instr,
8583                                struct instruction_data *data,
8584                                uint32_t n_instr)
8585 {
8586         uint32_t i;
8587
8588         for (i = 1; i < n_instr; i++) {
8589                 instr[0].type++;
8590                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
8591                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
8592                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
8593                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
8594
8595                 data[i].invalid = 1;
8596         }
8597 }
8598
8599 static uint32_t
8600 instr_pattern_dma_many_optimize(struct instruction *instructions,
8601                struct instruction_data *instruction_data,
8602                uint32_t n_instructions)
8603 {
8604         uint32_t i;
8605
8606         for (i = 0; i < n_instructions; ) {
8607                 struct instruction *instr = &instructions[i];
8608                 struct instruction_data *data = &instruction_data[i];
8609                 uint32_t n_instr = 0;
8610                 int detected;
8611
8612                 /* DMA many. */
8613                 detected = instr_pattern_dma_many_search(instr,
8614                                                          data,
8615                                                          n_instructions - i,
8616                                                          &n_instr);
8617                 if (detected) {
8618                         instr_pattern_dma_many_replace(instr, data, n_instr);
8619                         i += n_instr;
8620                         continue;
8621                 }
8622
8623                 /* No pattern starting at the current instruction. */
8624                 i++;
8625         }
8626
8627         /* Eliminate the invalid instructions that have been optimized out. */
8628         n_instructions = instr_compact(instructions,
8629                                        instruction_data,
8630                                        n_instructions);
8631
8632         return n_instructions;
8633 }
8634
8635 static uint32_t
8636 instr_optimize(struct rte_swx_pipeline *p,
8637                struct action *a,
8638                struct instruction *instructions,
8639                struct instruction_data *instruction_data,
8640                uint32_t n_instructions)
8641 {
8642         /* Extract many. */
8643         n_instructions = instr_pattern_extract_many_optimize(instructions,
8644                                                              instruction_data,
8645                                                              n_instructions);
8646
8647         /* Emit many + TX. */
8648         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
8649                                                              instruction_data,
8650                                                              n_instructions);
8651
8652         /* Mov all + validate. */
8653         n_instructions = instr_pattern_mov_all_validate_optimize(p,
8654                                                                  a,
8655                                                                  instructions,
8656                                                                  instruction_data,
8657                                                                  n_instructions);
8658
8659         /* DMA many. */
8660         n_instructions = instr_pattern_dma_many_optimize(instructions,
8661                                                          instruction_data,
8662                                                          n_instructions);
8663
8664         return n_instructions;
8665 }
8666
8667 static int
8668 instruction_config(struct rte_swx_pipeline *p,
8669                    struct action *a,
8670                    const char **instructions,
8671                    uint32_t n_instructions)
8672 {
8673         struct instruction *instr = NULL;
8674         struct instruction_data *data = NULL;
8675         int err = 0;
8676         uint32_t i;
8677
8678         CHECK(n_instructions, EINVAL);
8679         CHECK(instructions, EINVAL);
8680         for (i = 0; i < n_instructions; i++)
8681                 CHECK_INSTRUCTION(instructions[i], EINVAL);
8682
8683         /* Memory allocation. */
8684         instr = calloc(n_instructions, sizeof(struct instruction));
8685         if (!instr) {
8686                 err = -ENOMEM;
8687                 goto error;
8688         }
8689
8690         data = calloc(n_instructions, sizeof(struct instruction_data));
8691         if (!data) {
8692                 err = -ENOMEM;
8693                 goto error;
8694         }
8695
8696         for (i = 0; i < n_instructions; i++) {
8697                 char *string = strdup(instructions[i]);
8698                 if (!string) {
8699                         err = -ENOMEM;
8700                         goto error;
8701                 }
8702
8703                 err = instr_translate(p, a, string, &instr[i], &data[i]);
8704                 if (err) {
8705                         free(string);
8706                         goto error;
8707                 }
8708
8709                 free(string);
8710         }
8711
8712         err = instr_label_check(data, n_instructions);
8713         if (err)
8714                 goto error;
8715
8716         err = instr_verify(p, a, instr, data, n_instructions);
8717         if (err)
8718                 goto error;
8719
8720         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
8721
8722         err = instr_jmp_resolve(instr, data, n_instructions);
8723         if (err)
8724                 goto error;
8725
8726         if (a) {
8727                 a->instructions = instr;
8728                 a->n_instructions = n_instructions;
8729         } else {
8730                 p->instructions = instr;
8731                 p->n_instructions = n_instructions;
8732         }
8733
8734         free(data);
8735         return 0;
8736
8737 error:
8738         free(data);
8739         free(instr);
8740         return err;
8741 }
8742
8743 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
8744
8745 static instr_exec_t instruction_table[] = {
8746         [INSTR_RX] = instr_rx_exec,
8747         [INSTR_TX] = instr_tx_exec,
8748         [INSTR_TX_I] = instr_tx_i_exec,
8749
8750         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
8751         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
8752         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
8753         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
8754         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
8755         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
8756         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
8757         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
8758
8759         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
8760         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
8761         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
8762         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
8763         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
8764         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
8765         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
8766         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
8767         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
8768
8769         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
8770         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
8771
8772         [INSTR_MOV] = instr_mov_exec,
8773         [INSTR_MOV_MH] = instr_mov_mh_exec,
8774         [INSTR_MOV_HM] = instr_mov_hm_exec,
8775         [INSTR_MOV_HH] = instr_mov_hh_exec,
8776         [INSTR_MOV_I] = instr_mov_i_exec,
8777
8778         [INSTR_DMA_HT] = instr_dma_ht_exec,
8779         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
8780         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
8781         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
8782         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
8783         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
8784         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
8785         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
8786
8787         [INSTR_ALU_ADD] = instr_alu_add_exec,
8788         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
8789         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
8790         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
8791         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
8792         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
8793
8794         [INSTR_ALU_SUB] = instr_alu_sub_exec,
8795         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
8796         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
8797         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
8798         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
8799         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
8800
8801         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
8802         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
8803         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
8804         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
8805
8806         [INSTR_ALU_AND] = instr_alu_and_exec,
8807         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
8808         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
8809         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
8810         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
8811
8812         [INSTR_ALU_OR] = instr_alu_or_exec,
8813         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
8814         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
8815         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
8816         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
8817
8818         [INSTR_ALU_XOR] = instr_alu_xor_exec,
8819         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
8820         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
8821         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
8822         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
8823
8824         [INSTR_ALU_SHL] = instr_alu_shl_exec,
8825         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
8826         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
8827         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
8828         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
8829         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
8830
8831         [INSTR_ALU_SHR] = instr_alu_shr_exec,
8832         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
8833         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
8834         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
8835         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
8836         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
8837
8838         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
8839         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
8840         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
8841
8842         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
8843         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
8844         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
8845         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
8846         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
8847         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
8848
8849         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
8850         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
8851         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
8852         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
8853         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
8854         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
8855         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
8856         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
8857         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
8858
8859         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
8860         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
8861         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
8862         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
8863         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
8864         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
8865         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
8866         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
8867         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
8868
8869         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
8870         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
8871         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
8872
8873         [INSTR_METER_HHM] = instr_meter_hhm_exec,
8874         [INSTR_METER_HHI] = instr_meter_hhi_exec,
8875         [INSTR_METER_HMM] = instr_meter_hmm_exec,
8876         [INSTR_METER_HMI] = instr_meter_hmi_exec,
8877         [INSTR_METER_MHM] = instr_meter_mhm_exec,
8878         [INSTR_METER_MHI] = instr_meter_mhi_exec,
8879         [INSTR_METER_MMM] = instr_meter_mmm_exec,
8880         [INSTR_METER_MMI] = instr_meter_mmi_exec,
8881         [INSTR_METER_IHM] = instr_meter_ihm_exec,
8882         [INSTR_METER_IHI] = instr_meter_ihi_exec,
8883         [INSTR_METER_IMM] = instr_meter_imm_exec,
8884         [INSTR_METER_IMI] = instr_meter_imi_exec,
8885
8886         [INSTR_TABLE] = instr_table_exec,
8887         [INSTR_SELECTOR] = instr_selector_exec,
8888         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
8889         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
8890
8891         [INSTR_JMP] = instr_jmp_exec,
8892         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
8893         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
8894         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
8895         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
8896         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
8897         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
8898
8899         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
8900         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
8901         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
8902         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
8903         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
8904
8905         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
8906         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
8907         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
8908         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
8909         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
8910
8911         [INSTR_JMP_LT] = instr_jmp_lt_exec,
8912         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
8913         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
8914         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
8915         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
8916         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
8917
8918         [INSTR_JMP_GT] = instr_jmp_gt_exec,
8919         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
8920         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
8921         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
8922         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
8923         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
8924
8925         [INSTR_RETURN] = instr_return_exec,
8926 };
8927
8928 static inline void
8929 instr_exec(struct rte_swx_pipeline *p)
8930 {
8931         struct thread *t = &p->threads[p->thread_id];
8932         struct instruction *ip = t->ip;
8933         instr_exec_t instr = instruction_table[ip->type];
8934
8935         instr(p);
8936 }
8937
8938 /*
8939  * Action.
8940  */
8941 static struct action *
8942 action_find(struct rte_swx_pipeline *p, const char *name)
8943 {
8944         struct action *elem;
8945
8946         if (!name)
8947                 return NULL;
8948
8949         TAILQ_FOREACH(elem, &p->actions, node)
8950                 if (strcmp(elem->name, name) == 0)
8951                         return elem;
8952
8953         return NULL;
8954 }
8955
8956 static struct action *
8957 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8958 {
8959         struct action *action = NULL;
8960
8961         TAILQ_FOREACH(action, &p->actions, node)
8962                 if (action->id == id)
8963                         return action;
8964
8965         return NULL;
8966 }
8967
8968 static struct field *
8969 action_field_find(struct action *a, const char *name)
8970 {
8971         return a->st ? struct_type_field_find(a->st, name) : NULL;
8972 }
8973
8974 static struct field *
8975 action_field_parse(struct action *action, const char *name)
8976 {
8977         if (name[0] != 't' || name[1] != '.')
8978                 return NULL;
8979
8980         return action_field_find(action, &name[2]);
8981 }
8982
8983 int
8984 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
8985                                const char *name,
8986                                const char *args_struct_type_name,
8987                                const char **instructions,
8988                                uint32_t n_instructions)
8989 {
8990         struct struct_type *args_struct_type;
8991         struct action *a;
8992         int err;
8993
8994         CHECK(p, EINVAL);
8995
8996         CHECK_NAME(name, EINVAL);
8997         CHECK(!action_find(p, name), EEXIST);
8998
8999         if (args_struct_type_name) {
9000                 CHECK_NAME(args_struct_type_name, EINVAL);
9001                 args_struct_type = struct_type_find(p, args_struct_type_name);
9002                 CHECK(args_struct_type, EINVAL);
9003         } else {
9004                 args_struct_type = NULL;
9005         }
9006
9007         /* Node allocation. */
9008         a = calloc(1, sizeof(struct action));
9009         CHECK(a, ENOMEM);
9010         if (args_struct_type) {
9011                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
9012                 if (!a->args_endianness) {
9013                         free(a);
9014                         CHECK(0, ENOMEM);
9015                 }
9016         }
9017
9018         /* Node initialization. */
9019         strcpy(a->name, name);
9020         a->st = args_struct_type;
9021         a->id = p->n_actions;
9022
9023         /* Instruction translation. */
9024         err = instruction_config(p, a, instructions, n_instructions);
9025         if (err) {
9026                 free(a->args_endianness);
9027                 free(a);
9028                 return err;
9029         }
9030
9031         /* Node add to tailq. */
9032         TAILQ_INSERT_TAIL(&p->actions, a, node);
9033         p->n_actions++;
9034
9035         return 0;
9036 }
9037
9038 static int
9039 action_build(struct rte_swx_pipeline *p)
9040 {
9041         struct action *action;
9042
9043         p->action_instructions = calloc(p->n_actions,
9044                                         sizeof(struct instruction *));
9045         CHECK(p->action_instructions, ENOMEM);
9046
9047         TAILQ_FOREACH(action, &p->actions, node)
9048                 p->action_instructions[action->id] = action->instructions;
9049
9050         return 0;
9051 }
9052
9053 static void
9054 action_build_free(struct rte_swx_pipeline *p)
9055 {
9056         free(p->action_instructions);
9057         p->action_instructions = NULL;
9058 }
9059
9060 static void
9061 action_free(struct rte_swx_pipeline *p)
9062 {
9063         action_build_free(p);
9064
9065         for ( ; ; ) {
9066                 struct action *action;
9067
9068                 action = TAILQ_FIRST(&p->actions);
9069                 if (!action)
9070                         break;
9071
9072                 TAILQ_REMOVE(&p->actions, action, node);
9073                 free(action->instructions);
9074                 free(action);
9075         }
9076 }
9077
9078 static uint32_t
9079 action_arg_src_mov_count(struct action *a,
9080                          uint32_t arg_id,
9081                          struct instruction *instructions,
9082                          struct instruction_data *instruction_data,
9083                          uint32_t n_instructions)
9084 {
9085         uint32_t offset, n_users = 0, i;
9086
9087         if (!a->st ||
9088             (arg_id >= a->st->n_fields) ||
9089             !instructions ||
9090             !instruction_data ||
9091             !n_instructions)
9092                 return 0;
9093
9094         offset = a->st->fields[arg_id].offset / 8;
9095
9096         for (i = 0; i < n_instructions; i++) {
9097                 struct instruction *instr = &instructions[i];
9098                 struct instruction_data *data = &instruction_data[i];
9099
9100                 if (data->invalid ||
9101                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
9102                     instr->mov.src.struct_id ||
9103                     (instr->mov.src.offset != offset))
9104                         continue;
9105
9106                 n_users++;
9107         }
9108
9109         return n_users;
9110 }
9111
9112 /*
9113  * Table.
9114  */
9115 static struct table_type *
9116 table_type_find(struct rte_swx_pipeline *p, const char *name)
9117 {
9118         struct table_type *elem;
9119
9120         TAILQ_FOREACH(elem, &p->table_types, node)
9121                 if (strcmp(elem->name, name) == 0)
9122                         return elem;
9123
9124         return NULL;
9125 }
9126
9127 static struct table_type *
9128 table_type_resolve(struct rte_swx_pipeline *p,
9129                    const char *recommended_type_name,
9130                    enum rte_swx_table_match_type match_type)
9131 {
9132         struct table_type *elem;
9133
9134         /* Only consider the recommended type if the match type is correct. */
9135         if (recommended_type_name)
9136                 TAILQ_FOREACH(elem, &p->table_types, node)
9137                         if (!strcmp(elem->name, recommended_type_name) &&
9138                             (elem->match_type == match_type))
9139                                 return elem;
9140
9141         /* Ignore the recommended type and get the first element with this match
9142          * type.
9143          */
9144         TAILQ_FOREACH(elem, &p->table_types, node)
9145                 if (elem->match_type == match_type)
9146                         return elem;
9147
9148         return NULL;
9149 }
9150
9151 static struct table *
9152 table_find(struct rte_swx_pipeline *p, const char *name)
9153 {
9154         struct table *elem;
9155
9156         TAILQ_FOREACH(elem, &p->tables, node)
9157                 if (strcmp(elem->name, name) == 0)
9158                         return elem;
9159
9160         return NULL;
9161 }
9162
9163 static struct table *
9164 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9165 {
9166         struct table *table = NULL;
9167
9168         TAILQ_FOREACH(table, &p->tables, node)
9169                 if (table->id == id)
9170                         return table;
9171
9172         return NULL;
9173 }
9174
9175 int
9176 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
9177                                      const char *name,
9178                                      enum rte_swx_table_match_type match_type,
9179                                      struct rte_swx_table_ops *ops)
9180 {
9181         struct table_type *elem;
9182
9183         CHECK(p, EINVAL);
9184
9185         CHECK_NAME(name, EINVAL);
9186         CHECK(!table_type_find(p, name), EEXIST);
9187
9188         CHECK(ops, EINVAL);
9189         CHECK(ops->create, EINVAL);
9190         CHECK(ops->lkp, EINVAL);
9191         CHECK(ops->free, EINVAL);
9192
9193         /* Node allocation. */
9194         elem = calloc(1, sizeof(struct table_type));
9195         CHECK(elem, ENOMEM);
9196
9197         /* Node initialization. */
9198         strcpy(elem->name, name);
9199         elem->match_type = match_type;
9200         memcpy(&elem->ops, ops, sizeof(*ops));
9201
9202         /* Node add to tailq. */
9203         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
9204
9205         return 0;
9206 }
9207
9208 static int
9209 table_match_type_resolve(struct rte_swx_match_field_params *fields,
9210                          uint32_t n_fields,
9211                          enum rte_swx_table_match_type *match_type)
9212 {
9213         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
9214
9215         for (i = 0; i < n_fields; i++) {
9216                 struct rte_swx_match_field_params  *f = &fields[i];
9217
9218                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
9219                         n_fields_em++;
9220
9221                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
9222                         n_fields_lpm++;
9223         }
9224
9225         if ((n_fields_lpm > 1) ||
9226             (n_fields_lpm && (n_fields_em != n_fields - 1)))
9227                 return -EINVAL;
9228
9229         *match_type = (n_fields_em == n_fields) ?
9230                        RTE_SWX_TABLE_MATCH_EXACT :
9231                        RTE_SWX_TABLE_MATCH_WILDCARD;
9232
9233         return 0;
9234 }
9235
9236 static int
9237 table_match_fields_check(struct rte_swx_pipeline *p,
9238                          struct rte_swx_pipeline_table_params *params,
9239                          struct header **header)
9240 {
9241         struct header *h0 = NULL;
9242         struct field *hf, *mf;
9243         uint32_t *offset = NULL, i;
9244         int status = 0;
9245
9246         /* Return if no match fields. */
9247         if (!params->n_fields) {
9248                 if (params->fields) {
9249                         status = -EINVAL;
9250                         goto end;
9251                 }
9252
9253                 return 0;
9254         }
9255
9256         /* Memory allocation. */
9257         offset = calloc(params->n_fields, sizeof(uint32_t));
9258         if (!offset) {
9259                 status = -ENOMEM;
9260                 goto end;
9261         }
9262
9263         /* Check that all the match fields belong to either the same header or
9264          * to the meta-data.
9265          */
9266         hf = header_field_parse(p, params->fields[0].name, &h0);
9267         mf = metadata_field_parse(p, params->fields[0].name);
9268         if (!hf && !mf) {
9269                 status = -EINVAL;
9270                 goto end;
9271         }
9272
9273         offset[0] = h0 ? hf->offset : mf->offset;
9274
9275         for (i = 1; i < params->n_fields; i++)
9276                 if (h0) {
9277                         struct header *h;
9278
9279                         hf = header_field_parse(p, params->fields[i].name, &h);
9280                         if (!hf || (h->id != h0->id)) {
9281                                 status = -EINVAL;
9282                                 goto end;
9283                         }
9284
9285                         offset[i] = hf->offset;
9286                 } else {
9287                         mf = metadata_field_parse(p, params->fields[i].name);
9288                         if (!mf) {
9289                                 status = -EINVAL;
9290                                 goto end;
9291                         }
9292
9293                         offset[i] = mf->offset;
9294                 }
9295
9296         /* Check that there are no duplicated match fields. */
9297         for (i = 0; i < params->n_fields; i++) {
9298                 uint32_t j;
9299
9300                 for (j = 0; j < i; j++)
9301                         if (offset[j] == offset[i]) {
9302                                 status = -EINVAL;
9303                                 goto end;
9304                         }
9305         }
9306
9307         /* Return. */
9308         if (header)
9309                 *header = h0;
9310
9311 end:
9312         free(offset);
9313         return status;
9314 }
9315
9316 int
9317 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
9318                               const char *name,
9319                               struct rte_swx_pipeline_table_params *params,
9320                               const char *recommended_table_type_name,
9321                               const char *args,
9322                               uint32_t size)
9323 {
9324         struct table_type *type;
9325         struct table *t;
9326         struct action *default_action;
9327         struct header *header = NULL;
9328         uint32_t action_data_size_max = 0, i;
9329         int status = 0;
9330
9331         CHECK(p, EINVAL);
9332
9333         CHECK_NAME(name, EINVAL);
9334         CHECK(!table_find(p, name), EEXIST);
9335         CHECK(!selector_find(p, name), EEXIST);
9336
9337         CHECK(params, EINVAL);
9338
9339         /* Match checks. */
9340         status = table_match_fields_check(p, params, &header);
9341         if (status)
9342                 return status;
9343
9344         /* Action checks. */
9345         CHECK(params->n_actions, EINVAL);
9346         CHECK(params->action_names, EINVAL);
9347         for (i = 0; i < params->n_actions; i++) {
9348                 const char *action_name = params->action_names[i];
9349                 struct action *a;
9350                 uint32_t action_data_size;
9351
9352                 CHECK_NAME(action_name, EINVAL);
9353
9354                 a = action_find(p, action_name);
9355                 CHECK(a, EINVAL);
9356
9357                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
9358                 if (action_data_size > action_data_size_max)
9359                         action_data_size_max = action_data_size;
9360         }
9361
9362         CHECK_NAME(params->default_action_name, EINVAL);
9363         for (i = 0; i < p->n_actions; i++)
9364                 if (!strcmp(params->action_names[i],
9365                             params->default_action_name))
9366                         break;
9367         CHECK(i < params->n_actions, EINVAL);
9368         default_action = action_find(p, params->default_action_name);
9369         CHECK((default_action->st && params->default_action_data) ||
9370               !params->default_action_data, EINVAL);
9371
9372         /* Table type checks. */
9373         if (recommended_table_type_name)
9374                 CHECK_NAME(recommended_table_type_name, EINVAL);
9375
9376         if (params->n_fields) {
9377                 enum rte_swx_table_match_type match_type;
9378
9379                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
9380                 if (status)
9381                         return status;
9382
9383                 type = table_type_resolve(p, recommended_table_type_name, match_type);
9384                 CHECK(type, EINVAL);
9385         } else {
9386                 type = NULL;
9387         }
9388
9389         /* Memory allocation. */
9390         t = calloc(1, sizeof(struct table));
9391         CHECK(t, ENOMEM);
9392
9393         t->fields = calloc(params->n_fields, sizeof(struct match_field));
9394         if (!t->fields) {
9395                 free(t);
9396                 CHECK(0, ENOMEM);
9397         }
9398
9399         t->actions = calloc(params->n_actions, sizeof(struct action *));
9400         if (!t->actions) {
9401                 free(t->fields);
9402                 free(t);
9403                 CHECK(0, ENOMEM);
9404         }
9405
9406         if (action_data_size_max) {
9407                 t->default_action_data = calloc(1, action_data_size_max);
9408                 if (!t->default_action_data) {
9409                         free(t->actions);
9410                         free(t->fields);
9411                         free(t);
9412                         CHECK(0, ENOMEM);
9413                 }
9414         }
9415
9416         /* Node initialization. */
9417         strcpy(t->name, name);
9418         if (args && args[0])
9419                 strcpy(t->args, args);
9420         t->type = type;
9421
9422         for (i = 0; i < params->n_fields; i++) {
9423                 struct rte_swx_match_field_params *field = &params->fields[i];
9424                 struct match_field *f = &t->fields[i];
9425
9426                 f->match_type = field->match_type;
9427                 f->field = header ?
9428                         header_field_parse(p, field->name, NULL) :
9429                         metadata_field_parse(p, field->name);
9430         }
9431         t->n_fields = params->n_fields;
9432         t->header = header;
9433
9434         for (i = 0; i < params->n_actions; i++)
9435                 t->actions[i] = action_find(p, params->action_names[i]);
9436         t->default_action = default_action;
9437         if (default_action->st)
9438                 memcpy(t->default_action_data,
9439                        params->default_action_data,
9440                        default_action->st->n_bits / 8);
9441         t->n_actions = params->n_actions;
9442         t->default_action_is_const = params->default_action_is_const;
9443         t->action_data_size_max = action_data_size_max;
9444
9445         t->size = size;
9446         t->id = p->n_tables;
9447
9448         /* Node add to tailq. */
9449         TAILQ_INSERT_TAIL(&p->tables, t, node);
9450         p->n_tables++;
9451
9452         return 0;
9453 }
9454
9455 static struct rte_swx_table_params *
9456 table_params_get(struct table *table)
9457 {
9458         struct rte_swx_table_params *params;
9459         struct field *first, *last;
9460         uint8_t *key_mask;
9461         uint32_t key_size, key_offset, action_data_size, i;
9462
9463         /* Memory allocation. */
9464         params = calloc(1, sizeof(struct rte_swx_table_params));
9465         if (!params)
9466                 return NULL;
9467
9468         /* Find first (smallest offset) and last (biggest offset) match fields. */
9469         first = table->fields[0].field;
9470         last = table->fields[0].field;
9471
9472         for (i = 0; i < table->n_fields; i++) {
9473                 struct field *f = table->fields[i].field;
9474
9475                 if (f->offset < first->offset)
9476                         first = f;
9477
9478                 if (f->offset > last->offset)
9479                         last = f;
9480         }
9481
9482         /* Key offset and size. */
9483         key_offset = first->offset / 8;
9484         key_size = (last->offset + last->n_bits - first->offset) / 8;
9485
9486         /* Memory allocation. */
9487         key_mask = calloc(1, key_size);
9488         if (!key_mask) {
9489                 free(params);
9490                 return NULL;
9491         }
9492
9493         /* Key mask. */
9494         for (i = 0; i < table->n_fields; i++) {
9495                 struct field *f = table->fields[i].field;
9496                 uint32_t start = (f->offset - first->offset) / 8;
9497                 size_t size = f->n_bits / 8;
9498
9499                 memset(&key_mask[start], 0xFF, size);
9500         }
9501
9502         /* Action data size. */
9503         action_data_size = 0;
9504         for (i = 0; i < table->n_actions; i++) {
9505                 struct action *action = table->actions[i];
9506                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
9507
9508                 if (ads > action_data_size)
9509                         action_data_size = ads;
9510         }
9511
9512         /* Fill in. */
9513         params->match_type = table->type->match_type;
9514         params->key_size = key_size;
9515         params->key_offset = key_offset;
9516         params->key_mask0 = key_mask;
9517         params->action_data_size = action_data_size;
9518         params->n_keys_max = table->size;
9519
9520         return params;
9521 }
9522
9523 static void
9524 table_params_free(struct rte_swx_table_params *params)
9525 {
9526         if (!params)
9527                 return;
9528
9529         free(params->key_mask0);
9530         free(params);
9531 }
9532
9533 static int
9534 table_stub_lkp(void *table __rte_unused,
9535                void *mailbox __rte_unused,
9536                uint8_t **key __rte_unused,
9537                uint64_t *action_id __rte_unused,
9538                uint8_t **action_data __rte_unused,
9539                int *hit)
9540 {
9541         *hit = 0;
9542         return 1; /* DONE. */
9543 }
9544
9545 static int
9546 table_build(struct rte_swx_pipeline *p)
9547 {
9548         uint32_t i;
9549
9550         /* Per pipeline: table statistics. */
9551         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
9552         CHECK(p->table_stats, ENOMEM);
9553
9554         for (i = 0; i < p->n_tables; i++) {
9555                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9556                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
9557         }
9558
9559         /* Per thread: table runt-time. */
9560         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9561                 struct thread *t = &p->threads[i];
9562                 struct table *table;
9563
9564                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
9565                 CHECK(t->tables, ENOMEM);
9566
9567                 TAILQ_FOREACH(table, &p->tables, node) {
9568                         struct table_runtime *r = &t->tables[table->id];
9569
9570                         if (table->type) {
9571                                 uint64_t size;
9572
9573                                 size = table->type->ops.mailbox_size_get();
9574
9575                                 /* r->func. */
9576                                 r->func = table->type->ops.lkp;
9577
9578                                 /* r->mailbox. */
9579                                 if (size) {
9580                                         r->mailbox = calloc(1, size);
9581                                         CHECK(r->mailbox, ENOMEM);
9582                                 }
9583
9584                                 /* r->key. */
9585                                 r->key = table->header ?
9586                                         &t->structs[table->header->struct_id] :
9587                                         &t->structs[p->metadata_struct_id];
9588                         } else {
9589                                 r->func = table_stub_lkp;
9590                         }
9591                 }
9592         }
9593
9594         return 0;
9595 }
9596
9597 static void
9598 table_build_free(struct rte_swx_pipeline *p)
9599 {
9600         uint32_t i;
9601
9602         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9603                 struct thread *t = &p->threads[i];
9604                 uint32_t j;
9605
9606                 if (!t->tables)
9607                         continue;
9608
9609                 for (j = 0; j < p->n_tables; j++) {
9610                         struct table_runtime *r = &t->tables[j];
9611
9612                         free(r->mailbox);
9613                 }
9614
9615                 free(t->tables);
9616                 t->tables = NULL;
9617         }
9618
9619         if (p->table_stats) {
9620                 for (i = 0; i < p->n_tables; i++)
9621                         free(p->table_stats[i].n_pkts_action);
9622
9623                 free(p->table_stats);
9624         }
9625 }
9626
9627 static void
9628 table_free(struct rte_swx_pipeline *p)
9629 {
9630         table_build_free(p);
9631
9632         /* Tables. */
9633         for ( ; ; ) {
9634                 struct table *elem;
9635
9636                 elem = TAILQ_FIRST(&p->tables);
9637                 if (!elem)
9638                         break;
9639
9640                 TAILQ_REMOVE(&p->tables, elem, node);
9641                 free(elem->fields);
9642                 free(elem->actions);
9643                 free(elem->default_action_data);
9644                 free(elem);
9645         }
9646
9647         /* Table types. */
9648         for ( ; ; ) {
9649                 struct table_type *elem;
9650
9651                 elem = TAILQ_FIRST(&p->table_types);
9652                 if (!elem)
9653                         break;
9654
9655                 TAILQ_REMOVE(&p->table_types, elem, node);
9656                 free(elem);
9657         }
9658 }
9659
9660 /*
9661  * Selector.
9662  */
9663 static struct selector *
9664 selector_find(struct rte_swx_pipeline *p, const char *name)
9665 {
9666         struct selector *s;
9667
9668         TAILQ_FOREACH(s, &p->selectors, node)
9669                 if (strcmp(s->name, name) == 0)
9670                         return s;
9671
9672         return NULL;
9673 }
9674
9675 static struct selector *
9676 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9677 {
9678         struct selector *s = NULL;
9679
9680         TAILQ_FOREACH(s, &p->selectors, node)
9681                 if (s->id == id)
9682                         return s;
9683
9684         return NULL;
9685 }
9686
9687 static int
9688 selector_fields_check(struct rte_swx_pipeline *p,
9689                       struct rte_swx_pipeline_selector_params *params,
9690                       struct header **header)
9691 {
9692         struct header *h0 = NULL;
9693         struct field *hf, *mf;
9694         uint32_t i;
9695
9696         /* Return if no selector fields. */
9697         if (!params->n_selector_fields || !params->selector_field_names)
9698                 return -EINVAL;
9699
9700         /* Check that all the selector fields either belong to the same header
9701          * or are all meta-data fields.
9702          */
9703         hf = header_field_parse(p, params->selector_field_names[0], &h0);
9704         mf = metadata_field_parse(p, params->selector_field_names[0]);
9705         if (!hf && !mf)
9706                 return -EINVAL;
9707
9708         for (i = 1; i < params->n_selector_fields; i++)
9709                 if (h0) {
9710                         struct header *h;
9711
9712                         hf = header_field_parse(p, params->selector_field_names[i], &h);
9713                         if (!hf || (h->id != h0->id))
9714                                 return -EINVAL;
9715                 } else {
9716                         mf = metadata_field_parse(p, params->selector_field_names[i]);
9717                         if (!mf)
9718                                 return -EINVAL;
9719                 }
9720
9721         /* Check that there are no duplicated match fields. */
9722         for (i = 0; i < params->n_selector_fields; i++) {
9723                 const char *field_name = params->selector_field_names[i];
9724                 uint32_t j;
9725
9726                 for (j = i + 1; j < params->n_selector_fields; j++)
9727                         if (!strcmp(params->selector_field_names[j], field_name))
9728                                 return -EINVAL;
9729         }
9730
9731         /* Return. */
9732         if (header)
9733                 *header = h0;
9734
9735         return 0;
9736 }
9737
9738 int
9739 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
9740                                  const char *name,
9741                                  struct rte_swx_pipeline_selector_params *params)
9742 {
9743         struct selector *s;
9744         struct header *selector_header = NULL;
9745         struct field *group_id_field, *member_id_field;
9746         uint32_t i;
9747         int status = 0;
9748
9749         CHECK(p, EINVAL);
9750
9751         CHECK_NAME(name, EINVAL);
9752         CHECK(!table_find(p, name), EEXIST);
9753         CHECK(!selector_find(p, name), EEXIST);
9754
9755         CHECK(params, EINVAL);
9756
9757         CHECK_NAME(params->group_id_field_name, EINVAL);
9758         group_id_field = metadata_field_parse(p, params->group_id_field_name);
9759         CHECK(group_id_field, EINVAL);
9760
9761         for (i = 0; i < params->n_selector_fields; i++) {
9762                 const char *field_name = params->selector_field_names[i];
9763
9764                 CHECK_NAME(field_name, EINVAL);
9765         }
9766         status = selector_fields_check(p, params, &selector_header);
9767         if (status)
9768                 return status;
9769
9770         CHECK_NAME(params->member_id_field_name, EINVAL);
9771         member_id_field = metadata_field_parse(p, params->member_id_field_name);
9772         CHECK(member_id_field, EINVAL);
9773
9774         CHECK(params->n_groups_max, EINVAL);
9775
9776         CHECK(params->n_members_per_group_max, EINVAL);
9777
9778         /* Memory allocation. */
9779         s = calloc(1, sizeof(struct selector));
9780         if (!s) {
9781                 status = -ENOMEM;
9782                 goto error;
9783         }
9784
9785         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
9786         if (!s->selector_fields) {
9787                 status = -ENOMEM;
9788                 goto error;
9789         }
9790
9791         /* Node initialization. */
9792         strcpy(s->name, name);
9793
9794         s->group_id_field = group_id_field;
9795
9796         for (i = 0; i < params->n_selector_fields; i++) {
9797                 const char *field_name = params->selector_field_names[i];
9798
9799                 s->selector_fields[i] = selector_header ?
9800                         header_field_parse(p, field_name, NULL) :
9801                         metadata_field_parse(p, field_name);
9802         }
9803
9804         s->n_selector_fields = params->n_selector_fields;
9805
9806         s->selector_header = selector_header;
9807
9808         s->member_id_field = member_id_field;
9809
9810         s->n_groups_max = params->n_groups_max;
9811
9812         s->n_members_per_group_max = params->n_members_per_group_max;
9813
9814         s->id = p->n_selectors;
9815
9816         /* Node add to tailq. */
9817         TAILQ_INSERT_TAIL(&p->selectors, s, node);
9818         p->n_selectors++;
9819
9820         return 0;
9821
9822 error:
9823         if (!s)
9824                 return status;
9825
9826         free(s->selector_fields);
9827
9828         free(s);
9829
9830         return status;
9831 }
9832
9833 static void
9834 selector_params_free(struct rte_swx_table_selector_params *params)
9835 {
9836         if (!params)
9837                 return;
9838
9839         free(params->selector_mask);
9840
9841         free(params);
9842 }
9843
9844 static struct rte_swx_table_selector_params *
9845 selector_table_params_get(struct selector *s)
9846 {
9847         struct rte_swx_table_selector_params *params = NULL;
9848         struct field *first, *last;
9849         uint32_t i;
9850
9851         /* Memory allocation. */
9852         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
9853         if (!params)
9854                 goto error;
9855
9856         /* Group ID. */
9857         params->group_id_offset = s->group_id_field->offset / 8;
9858
9859         /* Find first (smallest offset) and last (biggest offset) selector fields. */
9860         first = s->selector_fields[0];
9861         last = s->selector_fields[0];
9862
9863         for (i = 0; i < s->n_selector_fields; i++) {
9864                 struct field *f = s->selector_fields[i];
9865
9866                 if (f->offset < first->offset)
9867                         first = f;
9868
9869                 if (f->offset > last->offset)
9870                         last = f;
9871         }
9872
9873         /* Selector offset and size. */
9874         params->selector_offset = first->offset / 8;
9875         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
9876
9877         /* Memory allocation. */
9878         params->selector_mask = calloc(1, params->selector_size);
9879         if (!params->selector_mask)
9880                 goto error;
9881
9882         /* Selector mask. */
9883         for (i = 0; i < s->n_selector_fields; i++) {
9884                 struct field *f = s->selector_fields[i];
9885                 uint32_t start = (f->offset - first->offset) / 8;
9886                 size_t size = f->n_bits / 8;
9887
9888                 memset(&params->selector_mask[start], 0xFF, size);
9889         }
9890
9891         /* Member ID. */
9892         params->member_id_offset = s->member_id_field->offset / 8;
9893
9894         /* Maximum number of groups. */
9895         params->n_groups_max = s->n_groups_max;
9896
9897         /* Maximum number of members per group. */
9898         params->n_members_per_group_max = s->n_members_per_group_max;
9899
9900         return params;
9901
9902 error:
9903         selector_params_free(params);
9904         return NULL;
9905 }
9906
9907 static void
9908 selector_build_free(struct rte_swx_pipeline *p)
9909 {
9910         uint32_t i;
9911
9912         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9913                 struct thread *t = &p->threads[i];
9914                 uint32_t j;
9915
9916                 if (!t->selectors)
9917                         continue;
9918
9919                 for (j = 0; j < p->n_selectors; j++) {
9920                         struct selector_runtime *r = &t->selectors[j];
9921
9922                         free(r->mailbox);
9923                 }
9924
9925                 free(t->selectors);
9926                 t->selectors = NULL;
9927         }
9928
9929         free(p->selector_stats);
9930         p->selector_stats = NULL;
9931 }
9932
9933 static int
9934 selector_build(struct rte_swx_pipeline *p)
9935 {
9936         uint32_t i;
9937         int status = 0;
9938
9939         /* Per pipeline: selector statistics. */
9940         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
9941         if (!p->selector_stats) {
9942                 status = -ENOMEM;
9943                 goto error;
9944         }
9945
9946         /* Per thread: selector run-time. */
9947         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9948                 struct thread *t = &p->threads[i];
9949                 struct selector *s;
9950
9951                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
9952                 if (!t->selectors) {
9953                         status = -ENOMEM;
9954                         goto error;
9955                 }
9956
9957                 TAILQ_FOREACH(s, &p->selectors, node) {
9958                         struct selector_runtime *r = &t->selectors[s->id];
9959                         uint64_t size;
9960
9961                         /* r->mailbox. */
9962                         size = rte_swx_table_selector_mailbox_size_get();
9963                         if (size) {
9964                                 r->mailbox = calloc(1, size);
9965                                 if (!r->mailbox) {
9966                                         status = -ENOMEM;
9967                                         goto error;
9968                                 }
9969                         }
9970
9971                         /* r->group_id_buffer. */
9972                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
9973
9974                         /* r->selector_buffer. */
9975                         r->selector_buffer = s->selector_header ?
9976                                 &t->structs[s->selector_header->struct_id] :
9977                                 &t->structs[p->metadata_struct_id];
9978
9979                         /* r->member_id_buffer. */
9980                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
9981                 }
9982         }
9983
9984         return 0;
9985
9986 error:
9987         selector_build_free(p);
9988         return status;
9989 }
9990
9991 static void
9992 selector_free(struct rte_swx_pipeline *p)
9993 {
9994         selector_build_free(p);
9995
9996         /* Selector tables. */
9997         for ( ; ; ) {
9998                 struct selector *elem;
9999
10000                 elem = TAILQ_FIRST(&p->selectors);
10001                 if (!elem)
10002                         break;
10003
10004                 TAILQ_REMOVE(&p->selectors, elem, node);
10005                 free(elem->selector_fields);
10006                 free(elem);
10007         }
10008 }
10009
10010 /*
10011  * Table state.
10012  */
10013 static int
10014 table_state_build(struct rte_swx_pipeline *p)
10015 {
10016         struct table *table;
10017         struct selector *s;
10018
10019         p->table_state = calloc(p->n_tables + p->n_selectors,
10020                                 sizeof(struct rte_swx_table_state));
10021         CHECK(p->table_state, ENOMEM);
10022
10023         TAILQ_FOREACH(table, &p->tables, node) {
10024                 struct rte_swx_table_state *ts = &p->table_state[table->id];
10025
10026                 if (table->type) {
10027                         struct rte_swx_table_params *params;
10028
10029                         /* ts->obj. */
10030                         params = table_params_get(table);
10031                         CHECK(params, ENOMEM);
10032
10033                         ts->obj = table->type->ops.create(params,
10034                                 NULL,
10035                                 table->args,
10036                                 p->numa_node);
10037
10038                         table_params_free(params);
10039                         CHECK(ts->obj, ENODEV);
10040                 }
10041
10042                 /* ts->default_action_data. */
10043                 if (table->action_data_size_max) {
10044                         ts->default_action_data =
10045                                 malloc(table->action_data_size_max);
10046                         CHECK(ts->default_action_data, ENOMEM);
10047
10048                         memcpy(ts->default_action_data,
10049                                table->default_action_data,
10050                                table->action_data_size_max);
10051                 }
10052
10053                 /* ts->default_action_id. */
10054                 ts->default_action_id = table->default_action->id;
10055         }
10056
10057         TAILQ_FOREACH(s, &p->selectors, node) {
10058                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
10059                 struct rte_swx_table_selector_params *params;
10060
10061                 /* ts->obj. */
10062                 params = selector_table_params_get(s);
10063                 CHECK(params, ENOMEM);
10064
10065                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
10066
10067                 selector_params_free(params);
10068                 CHECK(ts->obj, ENODEV);
10069         }
10070
10071         return 0;
10072 }
10073
10074 static void
10075 table_state_build_free(struct rte_swx_pipeline *p)
10076 {
10077         uint32_t i;
10078
10079         if (!p->table_state)
10080                 return;
10081
10082         for (i = 0; i < p->n_tables; i++) {
10083                 struct rte_swx_table_state *ts = &p->table_state[i];
10084                 struct table *table = table_find_by_id(p, i);
10085
10086                 /* ts->obj. */
10087                 if (table->type && ts->obj)
10088                         table->type->ops.free(ts->obj);
10089
10090                 /* ts->default_action_data. */
10091                 free(ts->default_action_data);
10092         }
10093
10094         for (i = 0; i < p->n_selectors; i++) {
10095                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
10096
10097                 /* ts->obj. */
10098                 if (ts->obj)
10099                         rte_swx_table_selector_free(ts->obj);
10100         }
10101
10102         free(p->table_state);
10103         p->table_state = NULL;
10104 }
10105
10106 static void
10107 table_state_free(struct rte_swx_pipeline *p)
10108 {
10109         table_state_build_free(p);
10110 }
10111
10112 /*
10113  * Register array.
10114  */
10115 static struct regarray *
10116 regarray_find(struct rte_swx_pipeline *p, const char *name)
10117 {
10118         struct regarray *elem;
10119
10120         TAILQ_FOREACH(elem, &p->regarrays, node)
10121                 if (!strcmp(elem->name, name))
10122                         return elem;
10123
10124         return NULL;
10125 }
10126
10127 static struct regarray *
10128 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
10129 {
10130         struct regarray *elem = NULL;
10131
10132         TAILQ_FOREACH(elem, &p->regarrays, node)
10133                 if (elem->id == id)
10134                         return elem;
10135
10136         return NULL;
10137 }
10138
10139 int
10140 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
10141                               const char *name,
10142                               uint32_t size,
10143                               uint64_t init_val)
10144 {
10145         struct regarray *r;
10146
10147         CHECK(p, EINVAL);
10148
10149         CHECK_NAME(name, EINVAL);
10150         CHECK(!regarray_find(p, name), EEXIST);
10151
10152         CHECK(size, EINVAL);
10153         size = rte_align32pow2(size);
10154
10155         /* Memory allocation. */
10156         r = calloc(1, sizeof(struct regarray));
10157         CHECK(r, ENOMEM);
10158
10159         /* Node initialization. */
10160         strcpy(r->name, name);
10161         r->init_val = init_val;
10162         r->size = size;
10163         r->id = p->n_regarrays;
10164
10165         /* Node add to tailq. */
10166         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
10167         p->n_regarrays++;
10168
10169         return 0;
10170 }
10171
10172 static int
10173 regarray_build(struct rte_swx_pipeline *p)
10174 {
10175         struct regarray *regarray;
10176
10177         if (!p->n_regarrays)
10178                 return 0;
10179
10180         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
10181         CHECK(p->regarray_runtime, ENOMEM);
10182
10183         TAILQ_FOREACH(regarray, &p->regarrays, node) {
10184                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
10185                 uint32_t i;
10186
10187                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
10188                                          RTE_CACHE_LINE_SIZE,
10189                                          p->numa_node);
10190                 CHECK(r->regarray, ENOMEM);
10191
10192                 if (regarray->init_val)
10193                         for (i = 0; i < regarray->size; i++)
10194                                 r->regarray[i] = regarray->init_val;
10195
10196                 r->size_mask = regarray->size - 1;
10197         }
10198
10199         return 0;
10200 }
10201
10202 static void
10203 regarray_build_free(struct rte_swx_pipeline *p)
10204 {
10205         uint32_t i;
10206
10207         if (!p->regarray_runtime)
10208                 return;
10209
10210         for (i = 0; i < p->n_regarrays; i++) {
10211                 struct regarray *regarray = regarray_find_by_id(p, i);
10212                 struct regarray_runtime *r = &p->regarray_runtime[i];
10213
10214                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
10215         }
10216
10217         free(p->regarray_runtime);
10218         p->regarray_runtime = NULL;
10219 }
10220
10221 static void
10222 regarray_free(struct rte_swx_pipeline *p)
10223 {
10224         regarray_build_free(p);
10225
10226         for ( ; ; ) {
10227                 struct regarray *elem;
10228
10229                 elem = TAILQ_FIRST(&p->regarrays);
10230                 if (!elem)
10231                         break;
10232
10233                 TAILQ_REMOVE(&p->regarrays, elem, node);
10234                 free(elem);
10235         }
10236 }
10237
10238 /*
10239  * Meter array.
10240  */
10241 static struct meter_profile *
10242 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
10243 {
10244         struct meter_profile *elem;
10245
10246         TAILQ_FOREACH(elem, &p->meter_profiles, node)
10247                 if (!strcmp(elem->name, name))
10248                         return elem;
10249
10250         return NULL;
10251 }
10252
10253 static struct metarray *
10254 metarray_find(struct rte_swx_pipeline *p, const char *name)
10255 {
10256         struct metarray *elem;
10257
10258         TAILQ_FOREACH(elem, &p->metarrays, node)
10259                 if (!strcmp(elem->name, name))
10260                         return elem;
10261
10262         return NULL;
10263 }
10264
10265 static struct metarray *
10266 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
10267 {
10268         struct metarray *elem = NULL;
10269
10270         TAILQ_FOREACH(elem, &p->metarrays, node)
10271                 if (elem->id == id)
10272                         return elem;
10273
10274         return NULL;
10275 }
10276
10277 int
10278 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
10279                                  const char *name,
10280                                  uint32_t size)
10281 {
10282         struct metarray *m;
10283
10284         CHECK(p, EINVAL);
10285
10286         CHECK_NAME(name, EINVAL);
10287         CHECK(!metarray_find(p, name), EEXIST);
10288
10289         CHECK(size, EINVAL);
10290         size = rte_align32pow2(size);
10291
10292         /* Memory allocation. */
10293         m = calloc(1, sizeof(struct metarray));
10294         CHECK(m, ENOMEM);
10295
10296         /* Node initialization. */
10297         strcpy(m->name, name);
10298         m->size = size;
10299         m->id = p->n_metarrays;
10300
10301         /* Node add to tailq. */
10302         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
10303         p->n_metarrays++;
10304
10305         return 0;
10306 }
10307
10308 struct meter_profile meter_profile_default = {
10309         .node = {0},
10310         .name = "",
10311         .params = {0},
10312
10313         .profile = {
10314                 .cbs = 10000,
10315                 .pbs = 10000,
10316                 .cir_period = 1,
10317                 .cir_bytes_per_period = 1,
10318                 .pir_period = 1,
10319                 .pir_bytes_per_period = 1,
10320         },
10321
10322         .n_users = 0,
10323 };
10324
10325 static void
10326 meter_init(struct meter *m)
10327 {
10328         memset(m, 0, sizeof(struct meter));
10329         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
10330         m->profile = &meter_profile_default;
10331         m->color_mask = RTE_COLOR_GREEN;
10332
10333         meter_profile_default.n_users++;
10334 }
10335
10336 static int
10337 metarray_build(struct rte_swx_pipeline *p)
10338 {
10339         struct metarray *m;
10340
10341         if (!p->n_metarrays)
10342                 return 0;
10343
10344         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
10345         CHECK(p->metarray_runtime, ENOMEM);
10346
10347         TAILQ_FOREACH(m, &p->metarrays, node) {
10348                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
10349                 uint32_t i;
10350
10351                 r->metarray = env_malloc(m->size * sizeof(struct meter),
10352                                          RTE_CACHE_LINE_SIZE,
10353                                          p->numa_node);
10354                 CHECK(r->metarray, ENOMEM);
10355
10356                 for (i = 0; i < m->size; i++)
10357                         meter_init(&r->metarray[i]);
10358
10359                 r->size_mask = m->size - 1;
10360         }
10361
10362         return 0;
10363 }
10364
10365 static void
10366 metarray_build_free(struct rte_swx_pipeline *p)
10367 {
10368         uint32_t i;
10369
10370         if (!p->metarray_runtime)
10371                 return;
10372
10373         for (i = 0; i < p->n_metarrays; i++) {
10374                 struct metarray *m = metarray_find_by_id(p, i);
10375                 struct metarray_runtime *r = &p->metarray_runtime[i];
10376
10377                 env_free(r->metarray, m->size * sizeof(struct meter));
10378         }
10379
10380         free(p->metarray_runtime);
10381         p->metarray_runtime = NULL;
10382 }
10383
10384 static void
10385 metarray_free(struct rte_swx_pipeline *p)
10386 {
10387         metarray_build_free(p);
10388
10389         /* Meter arrays. */
10390         for ( ; ; ) {
10391                 struct metarray *elem;
10392
10393                 elem = TAILQ_FIRST(&p->metarrays);
10394                 if (!elem)
10395                         break;
10396
10397                 TAILQ_REMOVE(&p->metarrays, elem, node);
10398                 free(elem);
10399         }
10400
10401         /* Meter profiles. */
10402         for ( ; ; ) {
10403                 struct meter_profile *elem;
10404
10405                 elem = TAILQ_FIRST(&p->meter_profiles);
10406                 if (!elem)
10407                         break;
10408
10409                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
10410                 free(elem);
10411         }
10412 }
10413
10414 /*
10415  * Pipeline.
10416  */
10417 int
10418 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
10419 {
10420         struct rte_swx_pipeline *pipeline;
10421
10422         /* Check input parameters. */
10423         CHECK(p, EINVAL);
10424
10425         /* Memory allocation. */
10426         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
10427         CHECK(pipeline, ENOMEM);
10428
10429         /* Initialization. */
10430         TAILQ_INIT(&pipeline->struct_types);
10431         TAILQ_INIT(&pipeline->port_in_types);
10432         TAILQ_INIT(&pipeline->ports_in);
10433         TAILQ_INIT(&pipeline->port_out_types);
10434         TAILQ_INIT(&pipeline->ports_out);
10435         TAILQ_INIT(&pipeline->extern_types);
10436         TAILQ_INIT(&pipeline->extern_objs);
10437         TAILQ_INIT(&pipeline->extern_funcs);
10438         TAILQ_INIT(&pipeline->headers);
10439         TAILQ_INIT(&pipeline->actions);
10440         TAILQ_INIT(&pipeline->table_types);
10441         TAILQ_INIT(&pipeline->tables);
10442         TAILQ_INIT(&pipeline->selectors);
10443         TAILQ_INIT(&pipeline->regarrays);
10444         TAILQ_INIT(&pipeline->meter_profiles);
10445         TAILQ_INIT(&pipeline->metarrays);
10446
10447         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
10448         pipeline->numa_node = numa_node;
10449
10450         *p = pipeline;
10451         return 0;
10452 }
10453
10454 void
10455 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
10456 {
10457         if (!p)
10458                 return;
10459
10460         free(p->instructions);
10461
10462         metarray_free(p);
10463         regarray_free(p);
10464         table_state_free(p);
10465         selector_free(p);
10466         table_free(p);
10467         action_free(p);
10468         metadata_free(p);
10469         header_free(p);
10470         extern_func_free(p);
10471         extern_obj_free(p);
10472         port_out_free(p);
10473         port_in_free(p);
10474         struct_free(p);
10475
10476         free(p);
10477 }
10478
10479 int
10480 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
10481                                      const char **instructions,
10482                                      uint32_t n_instructions)
10483 {
10484         int err;
10485         uint32_t i;
10486
10487         err = instruction_config(p, NULL, instructions, n_instructions);
10488         if (err)
10489                 return err;
10490
10491         /* Thread instruction pointer reset. */
10492         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10493                 struct thread *t = &p->threads[i];
10494
10495                 thread_ip_reset(p, t);
10496         }
10497
10498         return 0;
10499 }
10500
10501 int
10502 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
10503 {
10504         int status;
10505
10506         CHECK(p, EINVAL);
10507         CHECK(p->build_done == 0, EEXIST);
10508
10509         status = port_in_build(p);
10510         if (status)
10511                 goto error;
10512
10513         status = port_out_build(p);
10514         if (status)
10515                 goto error;
10516
10517         status = struct_build(p);
10518         if (status)
10519                 goto error;
10520
10521         status = extern_obj_build(p);
10522         if (status)
10523                 goto error;
10524
10525         status = extern_func_build(p);
10526         if (status)
10527                 goto error;
10528
10529         status = header_build(p);
10530         if (status)
10531                 goto error;
10532
10533         status = metadata_build(p);
10534         if (status)
10535                 goto error;
10536
10537         status = action_build(p);
10538         if (status)
10539                 goto error;
10540
10541         status = table_build(p);
10542         if (status)
10543                 goto error;
10544
10545         status = selector_build(p);
10546         if (status)
10547                 goto error;
10548
10549         status = table_state_build(p);
10550         if (status)
10551                 goto error;
10552
10553         status = regarray_build(p);
10554         if (status)
10555                 goto error;
10556
10557         status = metarray_build(p);
10558         if (status)
10559                 goto error;
10560
10561         p->build_done = 1;
10562         return 0;
10563
10564 error:
10565         metarray_build_free(p);
10566         regarray_build_free(p);
10567         table_state_build_free(p);
10568         selector_build_free(p);
10569         table_build_free(p);
10570         action_build_free(p);
10571         metadata_build_free(p);
10572         header_build_free(p);
10573         extern_func_build_free(p);
10574         extern_obj_build_free(p);
10575         port_out_build_free(p);
10576         port_in_build_free(p);
10577         struct_build_free(p);
10578
10579         return status;
10580 }
10581
10582 void
10583 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
10584 {
10585         uint32_t i;
10586
10587         for (i = 0; i < n_instructions; i++)
10588                 instr_exec(p);
10589 }
10590
10591 void
10592 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
10593 {
10594         uint32_t i;
10595
10596         for (i = 0; i < p->n_ports_out; i++) {
10597                 struct port_out_runtime *port = &p->out[i];
10598
10599                 if (port->flush)
10600                         port->flush(port->obj);
10601         }
10602 }
10603
10604 /*
10605  * Control.
10606  */
10607 int
10608 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
10609                               struct rte_swx_ctl_pipeline_info *pipeline)
10610 {
10611         struct action *action;
10612         struct table *table;
10613         uint32_t n_actions = 0, n_tables = 0;
10614
10615         if (!p || !pipeline)
10616                 return -EINVAL;
10617
10618         TAILQ_FOREACH(action, &p->actions, node)
10619                 n_actions++;
10620
10621         TAILQ_FOREACH(table, &p->tables, node)
10622                 n_tables++;
10623
10624         pipeline->n_ports_in = p->n_ports_in;
10625         pipeline->n_ports_out = p->n_ports_out;
10626         pipeline->n_actions = n_actions;
10627         pipeline->n_tables = n_tables;
10628         pipeline->n_selectors = p->n_selectors;
10629         pipeline->n_regarrays = p->n_regarrays;
10630         pipeline->n_metarrays = p->n_metarrays;
10631
10632         return 0;
10633 }
10634
10635 int
10636 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
10637 {
10638         if (!p || !numa_node)
10639                 return -EINVAL;
10640
10641         *numa_node = p->numa_node;
10642         return 0;
10643 }
10644
10645 int
10646 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
10647                             uint32_t action_id,
10648                             struct rte_swx_ctl_action_info *action)
10649 {
10650         struct action *a = NULL;
10651
10652         if (!p || (action_id >= p->n_actions) || !action)
10653                 return -EINVAL;
10654
10655         a = action_find_by_id(p, action_id);
10656         if (!a)
10657                 return -EINVAL;
10658
10659         strcpy(action->name, a->name);
10660         action->n_args = a->st ? a->st->n_fields : 0;
10661         return 0;
10662 }
10663
10664 int
10665 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10666                                 uint32_t action_id,
10667                                 uint32_t action_arg_id,
10668                                 struct rte_swx_ctl_action_arg_info *action_arg)
10669 {
10670         struct action *a = NULL;
10671         struct field *arg = NULL;
10672
10673         if (!p || (action_id >= p->n_actions) || !action_arg)
10674                 return -EINVAL;
10675
10676         a = action_find_by_id(p, action_id);
10677         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10678                 return -EINVAL;
10679
10680         arg = &a->st->fields[action_arg_id];
10681         strcpy(action_arg->name, arg->name);
10682         action_arg->n_bits = arg->n_bits;
10683         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10684
10685         return 0;
10686 }
10687
10688 int
10689 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10690                            uint32_t table_id,
10691                            struct rte_swx_ctl_table_info *table)
10692 {
10693         struct table *t = NULL;
10694
10695         if (!p || !table)
10696                 return -EINVAL;
10697
10698         t = table_find_by_id(p, table_id);
10699         if (!t)
10700                 return -EINVAL;
10701
10702         strcpy(table->name, t->name);
10703         strcpy(table->args, t->args);
10704         table->n_match_fields = t->n_fields;
10705         table->n_actions = t->n_actions;
10706         table->default_action_is_const = t->default_action_is_const;
10707         table->size = t->size;
10708         return 0;
10709 }
10710
10711 int
10712 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10713         uint32_t table_id,
10714         uint32_t match_field_id,
10715         struct rte_swx_ctl_table_match_field_info *match_field)
10716 {
10717         struct table *t;
10718         struct match_field *f;
10719
10720         if (!p || (table_id >= p->n_tables) || !match_field)
10721                 return -EINVAL;
10722
10723         t = table_find_by_id(p, table_id);
10724         if (!t || (match_field_id >= t->n_fields))
10725                 return -EINVAL;
10726
10727         f = &t->fields[match_field_id];
10728         match_field->match_type = f->match_type;
10729         match_field->is_header = t->header ? 1 : 0;
10730         match_field->n_bits = f->field->n_bits;
10731         match_field->offset = f->field->offset;
10732
10733         return 0;
10734 }
10735
10736 int
10737 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10738         uint32_t table_id,
10739         uint32_t table_action_id,
10740         struct rte_swx_ctl_table_action_info *table_action)
10741 {
10742         struct table *t;
10743
10744         if (!p || (table_id >= p->n_tables) || !table_action)
10745                 return -EINVAL;
10746
10747         t = table_find_by_id(p, table_id);
10748         if (!t || (table_action_id >= t->n_actions))
10749                 return -EINVAL;
10750
10751         table_action->action_id = t->actions[table_action_id]->id;
10752
10753         return 0;
10754 }
10755
10756 int
10757 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10758                           uint32_t table_id,
10759                           struct rte_swx_table_ops *table_ops,
10760                           int *is_stub)
10761 {
10762         struct table *t;
10763
10764         if (!p || (table_id >= p->n_tables))
10765                 return -EINVAL;
10766
10767         t = table_find_by_id(p, table_id);
10768         if (!t)
10769                 return -EINVAL;
10770
10771         if (t->type) {
10772                 if (table_ops)
10773                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10774                 *is_stub = 0;
10775         } else {
10776                 *is_stub = 1;
10777         }
10778
10779         return 0;
10780 }
10781
10782 int
10783 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10784                               uint32_t selector_id,
10785                               struct rte_swx_ctl_selector_info *selector)
10786 {
10787         struct selector *s = NULL;
10788
10789         if (!p || !selector)
10790                 return -EINVAL;
10791
10792         s = selector_find_by_id(p, selector_id);
10793         if (!s)
10794                 return -EINVAL;
10795
10796         strcpy(selector->name, s->name);
10797
10798         selector->n_selector_fields = s->n_selector_fields;
10799         selector->n_groups_max = s->n_groups_max;
10800         selector->n_members_per_group_max = s->n_members_per_group_max;
10801
10802         return 0;
10803 }
10804
10805 int
10806 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10807          uint32_t selector_id,
10808          struct rte_swx_ctl_table_match_field_info *field)
10809 {
10810         struct selector *s;
10811
10812         if (!p || (selector_id >= p->n_selectors) || !field)
10813                 return -EINVAL;
10814
10815         s = selector_find_by_id(p, selector_id);
10816         if (!s)
10817                 return -EINVAL;
10818
10819         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10820         field->is_header = 0;
10821         field->n_bits = s->group_id_field->n_bits;
10822         field->offset = s->group_id_field->offset;
10823
10824         return 0;
10825 }
10826
10827 int
10828 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10829          uint32_t selector_id,
10830          uint32_t selector_field_id,
10831          struct rte_swx_ctl_table_match_field_info *field)
10832 {
10833         struct selector *s;
10834         struct field *f;
10835
10836         if (!p || (selector_id >= p->n_selectors) || !field)
10837                 return -EINVAL;
10838
10839         s = selector_find_by_id(p, selector_id);
10840         if (!s || (selector_field_id >= s->n_selector_fields))
10841                 return -EINVAL;
10842
10843         f = s->selector_fields[selector_field_id];
10844         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10845         field->is_header = s->selector_header ? 1 : 0;
10846         field->n_bits = f->n_bits;
10847         field->offset = f->offset;
10848
10849         return 0;
10850 }
10851
10852 int
10853 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10854          uint32_t selector_id,
10855          struct rte_swx_ctl_table_match_field_info *field)
10856 {
10857         struct selector *s;
10858
10859         if (!p || (selector_id >= p->n_selectors) || !field)
10860                 return -EINVAL;
10861
10862         s = selector_find_by_id(p, selector_id);
10863         if (!s)
10864                 return -EINVAL;
10865
10866         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10867         field->is_header = 0;
10868         field->n_bits = s->member_id_field->n_bits;
10869         field->offset = s->member_id_field->offset;
10870
10871         return 0;
10872 }
10873
10874 int
10875 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10876                                  struct rte_swx_table_state **table_state)
10877 {
10878         if (!p || !table_state || !p->build_done)
10879                 return -EINVAL;
10880
10881         *table_state = p->table_state;
10882         return 0;
10883 }
10884
10885 int
10886 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10887                                  struct rte_swx_table_state *table_state)
10888 {
10889         if (!p || !table_state || !p->build_done)
10890                 return -EINVAL;
10891
10892         p->table_state = table_state;
10893         return 0;
10894 }
10895
10896 int
10897 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10898                                         uint32_t port_id,
10899                                         struct rte_swx_port_in_stats *stats)
10900 {
10901         struct port_in *port;
10902
10903         if (!p || !stats)
10904                 return -EINVAL;
10905
10906         port = port_in_find(p, port_id);
10907         if (!port)
10908                 return -EINVAL;
10909
10910         port->type->ops.stats_read(port->obj, stats);
10911         return 0;
10912 }
10913
10914 int
10915 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10916                                          uint32_t port_id,
10917                                          struct rte_swx_port_out_stats *stats)
10918 {
10919         struct port_out *port;
10920
10921         if (!p || !stats)
10922                 return -EINVAL;
10923
10924         port = port_out_find(p, port_id);
10925         if (!port)
10926                 return -EINVAL;
10927
10928         port->type->ops.stats_read(port->obj, stats);
10929         return 0;
10930 }
10931
10932 int
10933 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10934                                       const char *table_name,
10935                                       struct rte_swx_table_stats *stats)
10936 {
10937         struct table *table;
10938         struct table_statistics *table_stats;
10939
10940         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10941                 return -EINVAL;
10942
10943         table = table_find(p, table_name);
10944         if (!table)
10945                 return -EINVAL;
10946
10947         table_stats = &p->table_stats[table->id];
10948
10949         memcpy(stats->n_pkts_action,
10950                table_stats->n_pkts_action,
10951                p->n_actions * sizeof(uint64_t));
10952
10953         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10954         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10955
10956         return 0;
10957 }
10958
10959 int
10960 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10961         const char *selector_name,
10962         struct rte_swx_pipeline_selector_stats *stats)
10963 {
10964         struct selector *s;
10965
10966         if (!p || !selector_name || !selector_name[0] || !stats)
10967                 return -EINVAL;
10968
10969         s = selector_find(p, selector_name);
10970         if (!s)
10971                 return -EINVAL;
10972
10973         stats->n_pkts = p->selector_stats[s->id].n_pkts;
10974
10975         return 0;
10976 }
10977
10978 int
10979 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10980                               uint32_t regarray_id,
10981                               struct rte_swx_ctl_regarray_info *regarray)
10982 {
10983         struct regarray *r;
10984
10985         if (!p || !regarray)
10986                 return -EINVAL;
10987
10988         r = regarray_find_by_id(p, regarray_id);
10989         if (!r)
10990                 return -EINVAL;
10991
10992         strcpy(regarray->name, r->name);
10993         regarray->size = r->size;
10994         return 0;
10995 }
10996
10997 int
10998 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10999                                    const char *regarray_name,
11000                                    uint32_t regarray_index,
11001                                    uint64_t *value)
11002 {
11003         struct regarray *regarray;
11004         struct regarray_runtime *r;
11005
11006         if (!p || !regarray_name || !value)
11007                 return -EINVAL;
11008
11009         regarray = regarray_find(p, regarray_name);
11010         if (!regarray || (regarray_index >= regarray->size))
11011                 return -EINVAL;
11012
11013         r = &p->regarray_runtime[regarray->id];
11014         *value = r->regarray[regarray_index];
11015         return 0;
11016 }
11017
11018 int
11019 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
11020                                    const char *regarray_name,
11021                                    uint32_t regarray_index,
11022                                    uint64_t value)
11023 {
11024         struct regarray *regarray;
11025         struct regarray_runtime *r;
11026
11027         if (!p || !regarray_name)
11028                 return -EINVAL;
11029
11030         regarray = regarray_find(p, regarray_name);
11031         if (!regarray || (regarray_index >= regarray->size))
11032                 return -EINVAL;
11033
11034         r = &p->regarray_runtime[regarray->id];
11035         r->regarray[regarray_index] = value;
11036         return 0;
11037 }
11038
11039 int
11040 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
11041                               uint32_t metarray_id,
11042                               struct rte_swx_ctl_metarray_info *metarray)
11043 {
11044         struct metarray *m;
11045
11046         if (!p || !metarray)
11047                 return -EINVAL;
11048
11049         m = metarray_find_by_id(p, metarray_id);
11050         if (!m)
11051                 return -EINVAL;
11052
11053         strcpy(metarray->name, m->name);
11054         metarray->size = m->size;
11055         return 0;
11056 }
11057
11058 int
11059 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
11060                               const char *name,
11061                               struct rte_meter_trtcm_params *params)
11062 {
11063         struct meter_profile *mp;
11064         int status;
11065
11066         CHECK(p, EINVAL);
11067         CHECK_NAME(name, EINVAL);
11068         CHECK(params, EINVAL);
11069         CHECK(!meter_profile_find(p, name), EEXIST);
11070
11071         /* Node allocation. */
11072         mp = calloc(1, sizeof(struct meter_profile));
11073         CHECK(mp, ENOMEM);
11074
11075         /* Node initialization. */
11076         strcpy(mp->name, name);
11077         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
11078         status = rte_meter_trtcm_profile_config(&mp->profile, params);
11079         if (status) {
11080                 free(mp);
11081                 CHECK(0, EINVAL);
11082         }
11083
11084         /* Node add to tailq. */
11085         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
11086
11087         return 0;
11088 }
11089
11090 int
11091 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
11092                                  const char *name)
11093 {
11094         struct meter_profile *mp;
11095
11096         CHECK(p, EINVAL);
11097         CHECK_NAME(name, EINVAL);
11098
11099         mp = meter_profile_find(p, name);
11100         CHECK(mp, EINVAL);
11101         CHECK(!mp->n_users, EBUSY);
11102
11103         /* Remove node from tailq. */
11104         TAILQ_REMOVE(&p->meter_profiles, mp, node);
11105         free(mp);
11106
11107         return 0;
11108 }
11109
11110 int
11111 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
11112                         const char *metarray_name,
11113                         uint32_t metarray_index)
11114 {
11115         struct meter_profile *mp_old;
11116         struct metarray *metarray;
11117         struct metarray_runtime *metarray_runtime;
11118         struct meter *m;
11119
11120         CHECK(p, EINVAL);
11121         CHECK_NAME(metarray_name, EINVAL);
11122
11123         metarray = metarray_find(p, metarray_name);
11124         CHECK(metarray, EINVAL);
11125         CHECK(metarray_index < metarray->size, EINVAL);
11126
11127         metarray_runtime = &p->metarray_runtime[metarray->id];
11128         m = &metarray_runtime->metarray[metarray_index];
11129         mp_old = m->profile;
11130
11131         meter_init(m);
11132
11133         mp_old->n_users--;
11134
11135         return 0;
11136 }
11137
11138 int
11139 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
11140                       const char *metarray_name,
11141                       uint32_t metarray_index,
11142                       const char *profile_name)
11143 {
11144         struct meter_profile *mp, *mp_old;
11145         struct metarray *metarray;
11146         struct metarray_runtime *metarray_runtime;
11147         struct meter *m;
11148
11149         CHECK(p, EINVAL);
11150         CHECK_NAME(metarray_name, EINVAL);
11151
11152         metarray = metarray_find(p, metarray_name);
11153         CHECK(metarray, EINVAL);
11154         CHECK(metarray_index < metarray->size, EINVAL);
11155
11156         mp = meter_profile_find(p, profile_name);
11157         CHECK(mp, EINVAL);
11158
11159         metarray_runtime = &p->metarray_runtime[metarray->id];
11160         m = &metarray_runtime->metarray[metarray_index];
11161         mp_old = m->profile;
11162
11163         memset(m, 0, sizeof(struct meter));
11164         rte_meter_trtcm_config(&m->m, &mp->profile);
11165         m->profile = mp;
11166         m->color_mask = RTE_COLORS;
11167
11168         mp->n_users++;
11169         mp_old->n_users--;
11170
11171         return 0;
11172 }
11173
11174 int
11175 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
11176                              const char *metarray_name,
11177                              uint32_t metarray_index,
11178                              struct rte_swx_ctl_meter_stats *stats)
11179 {
11180         struct metarray *metarray;
11181         struct metarray_runtime *metarray_runtime;
11182         struct meter *m;
11183
11184         CHECK(p, EINVAL);
11185         CHECK_NAME(metarray_name, EINVAL);
11186
11187         metarray = metarray_find(p, metarray_name);
11188         CHECK(metarray, EINVAL);
11189         CHECK(metarray_index < metarray->size, EINVAL);
11190
11191         CHECK(stats, EINVAL);
11192
11193         metarray_runtime = &p->metarray_runtime[metarray->id];
11194         m = &metarray_runtime->metarray[metarray_index];
11195
11196         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
11197         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
11198
11199         return 0;
11200 }