lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <inttypes.h>
   9 #include <sys/queue.h>
  10 #include <arpa/inet.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_prefetch.h>
  14 #include <rte_byteorder.h>
  15 #include <rte_cycles.h>
  16 #include <rte_meter.h>
  17
  18 #include <rte_swx_table_selector.h>
  19
  20 #include "rte_swx_pipeline.h"
  21 #include "rte_swx_ctl.h"
  22
  23 #define CHECK(condition, err_code)                                             \
  24 do {                                                                           \
  25         if (!(condition))                                                      \
  26                 return -(err_code);                                            \
  27 } while (0)
  28
  29 #define CHECK_NAME(name, err_code)                                             \
  30         CHECK((name) &&                                                        \
  31               (name)[0] &&                                                     \
  32               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  33               err_code)
  34
  35 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  36         CHECK((instr) &&                                                       \
  37               (instr)[0] &&                                                    \
  38               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  39                RTE_SWX_INSTRUCTION_SIZE),                                      \
  40               err_code)
  41
  42 #ifndef TRACE_LEVEL
  43 #define TRACE_LEVEL 0
  44 #endif
  45
  46 #if TRACE_LEVEL
  47 #define TRACE(...) printf(__VA_ARGS__)
  48 #else
  49 #define TRACE(...)
  50 #endif
  51
  52 /*
  53  * Environment.
  54  */
  55 #define ntoh64(x) rte_be_to_cpu_64(x)
  56 #define hton64(x) rte_cpu_to_be_64(x)
  57
  58 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  59
  60 #include <rte_malloc.h>
  61
  62 static void *
  63 env_malloc(size_t size, size_t alignment, int numa_node)
  64 {
  65         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  66 }
  67
  68 static void
  69 env_free(void *start, size_t size __rte_unused)
  70 {
  71         rte_free(start);
  72 }
  73
  74 #else
  75
  76 #include <numa.h>
  77
  78 static void *
  79 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  80 {
  81         void *start;
  82
  83         if (numa_available() == -1)
  84                 return NULL;
  85
  86         start = numa_alloc_onnode(size, numa_node);
  87         if (!start)
  88                 return NULL;
  89
  90         memset(start, 0, size);
  91         return start;
  92 }
  93
  94 static void
  95 env_free(void *start, size_t size)
  96 {
  97         if (numa_available() == -1)
  98                 return;
  99
 100         numa_free(start, size);
 101 }
 102
 103 #endif
 104
 105 /*
 106  * Struct.
 107  */
 108 struct field {
 109         char name[RTE_SWX_NAME_SIZE];
 110         uint32_t n_bits;
 111         uint32_t offset;
 112         int var_size;
 113 };
 114
 115 struct struct_type {
 116         TAILQ_ENTRY(struct_type) node;
 117         char name[RTE_SWX_NAME_SIZE];
 118         struct field *fields;
 119         uint32_t n_fields;
 120         uint32_t n_bits;
 121         uint32_t n_bits_min;
 122         int var_size;
 123 };
 124
 125 TAILQ_HEAD(struct_type_tailq, struct_type);
 126
 127 /*
 128  * Input port.
 129  */
 130 struct port_in_type {
 131         TAILQ_ENTRY(port_in_type) node;
 132         char name[RTE_SWX_NAME_SIZE];
 133         struct rte_swx_port_in_ops ops;
 134 };
 135
 136 TAILQ_HEAD(port_in_type_tailq, port_in_type);
 137
 138 struct port_in {
 139         TAILQ_ENTRY(port_in) node;
 140         struct port_in_type *type;
 141         void *obj;
 142         uint32_t id;
 143 };
 144
 145 TAILQ_HEAD(port_in_tailq, port_in);
 146
 147 struct port_in_runtime {
 148         rte_swx_port_in_pkt_rx_t pkt_rx;
 149         void *obj;
 150 };
 151
 152 /*
 153  * Output port.
 154  */
 155 struct port_out_type {
 156         TAILQ_ENTRY(port_out_type) node;
 157         char name[RTE_SWX_NAME_SIZE];
 158         struct rte_swx_port_out_ops ops;
 159 };
 160
 161 TAILQ_HEAD(port_out_type_tailq, port_out_type);
 162
 163 struct port_out {
 164         TAILQ_ENTRY(port_out) node;
 165         struct port_out_type *type;
 166         void *obj;
 167         uint32_t id;
 168 };
 169
 170 TAILQ_HEAD(port_out_tailq, port_out);
 171
 172 struct port_out_runtime {
 173         rte_swx_port_out_pkt_tx_t pkt_tx;
 174         rte_swx_port_out_flush_t flush;
 175         void *obj;
 176 };
 177
 178 /*
 179  * Extern object.
 180  */
 181 struct extern_type_member_func {
 182         TAILQ_ENTRY(extern_type_member_func) node;
 183         char name[RTE_SWX_NAME_SIZE];
 184         rte_swx_extern_type_member_func_t func;
 185         uint32_t id;
 186 };
 187
 188 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 189
 190 struct extern_type {
 191         TAILQ_ENTRY(extern_type) node;
 192         char name[RTE_SWX_NAME_SIZE];
 193         struct struct_type *mailbox_struct_type;
 194         rte_swx_extern_type_constructor_t constructor;
 195         rte_swx_extern_type_destructor_t destructor;
 196         struct extern_type_member_func_tailq funcs;
 197         uint32_t n_funcs;
 198 };
 199
 200 TAILQ_HEAD(extern_type_tailq, extern_type);
 201
 202 struct extern_obj {
 203         TAILQ_ENTRY(extern_obj) node;
 204         char name[RTE_SWX_NAME_SIZE];
 205         struct extern_type *type;
 206         void *obj;
 207         uint32_t struct_id;
 208         uint32_t id;
 209 };
 210
 211 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 212
 213 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 214 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 215 #endif
 216
 217 struct extern_obj_runtime {
 218         void *obj;
 219         uint8_t *mailbox;
 220         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 221 };
 222
 223 /*
 224  * Extern function.
 225  */
 226 struct extern_func {
 227         TAILQ_ENTRY(extern_func) node;
 228         char name[RTE_SWX_NAME_SIZE];
 229         struct struct_type *mailbox_struct_type;
 230         rte_swx_extern_func_t func;
 231         uint32_t struct_id;
 232         uint32_t id;
 233 };
 234
 235 TAILQ_HEAD(extern_func_tailq, extern_func);
 236
 237 struct extern_func_runtime {
 238         uint8_t *mailbox;
 239         rte_swx_extern_func_t func;
 240 };
 241
 242 /*
 243  * Header.
 244  */
 245 struct header {
 246         TAILQ_ENTRY(header) node;
 247         char name[RTE_SWX_NAME_SIZE];
 248         struct struct_type *st;
 249         uint32_t struct_id;
 250         uint32_t id;
 251 };
 252
 253 TAILQ_HEAD(header_tailq, header);
 254
 255 struct header_runtime {
 256         uint8_t *ptr0;
 257         uint32_t n_bytes;
 258 };
 259
 260 struct header_out_runtime {
 261         uint8_t *ptr0;
 262         uint8_t *ptr;
 263         uint32_t n_bytes;
 264 };
 265
 266 /*
 267  * Instruction.
 268  */
 269
 270 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 271  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 272  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 273  * when transferred to packet meta-data and in NBO when transferred to packet
 274  * headers.
 275  */
 276
 277 /* Notation conventions:
 278  *    -Header field: H = h.header.field (dst/src)
 279  *    -Meta-data field: M = m.field (dst/src)
 280  *    -Extern object mailbox field: E = e.field (dst/src)
 281  *    -Extern function mailbox field: F = f.field (dst/src)
 282  *    -Table action data field: T = t.field (src only)
 283  *    -Immediate value: I = 32-bit unsigned value (src only)
 284  */
 285
 286 enum instruction_type {
 287         /* rx m.port_in */
 288         INSTR_RX,
 289
 290         /* tx port_out
 291          * port_out = MI
 292          */
 293         INSTR_TX,   /* port_out = M */
 294         INSTR_TX_I, /* port_out = I */
 295
 296         /* extract h.header */
 297         INSTR_HDR_EXTRACT,
 298         INSTR_HDR_EXTRACT2,
 299         INSTR_HDR_EXTRACT3,
 300         INSTR_HDR_EXTRACT4,
 301         INSTR_HDR_EXTRACT5,
 302         INSTR_HDR_EXTRACT6,
 303         INSTR_HDR_EXTRACT7,
 304         INSTR_HDR_EXTRACT8,
 305
 306         /* extract h.header m.last_field_size */
 307         INSTR_HDR_EXTRACT_M,
 308
 309         /* lookahead h.header */
 310         INSTR_HDR_LOOKAHEAD,
 311
 312         /* emit h.header */
 313         INSTR_HDR_EMIT,
 314         INSTR_HDR_EMIT_TX,
 315         INSTR_HDR_EMIT2_TX,
 316         INSTR_HDR_EMIT3_TX,
 317         INSTR_HDR_EMIT4_TX,
 318         INSTR_HDR_EMIT5_TX,
 319         INSTR_HDR_EMIT6_TX,
 320         INSTR_HDR_EMIT7_TX,
 321         INSTR_HDR_EMIT8_TX,
 322
 323         /* validate h.header */
 324         INSTR_HDR_VALIDATE,
 325
 326         /* invalidate h.header */
 327         INSTR_HDR_INVALIDATE,
 328
 329         /* mov dst src
 330          * dst = src
 331          * dst = HMEF, src = HMEFTI
 332          */
 333         INSTR_MOV,    /* dst = MEF, src = MEFT */
 334         INSTR_MOV_MH, /* dst = MEF, src = H */
 335         INSTR_MOV_HM, /* dst = H, src = MEFT */
 336         INSTR_MOV_HH, /* dst = H, src = H */
 337         INSTR_MOV_I,  /* dst = HMEF, src = I */
 338
 339         /* dma h.header t.field
 340          * memcpy(h.header, t.field, sizeof(h.header))
 341          */
 342         INSTR_DMA_HT,
 343         INSTR_DMA_HT2,
 344         INSTR_DMA_HT3,
 345         INSTR_DMA_HT4,
 346         INSTR_DMA_HT5,
 347         INSTR_DMA_HT6,
 348         INSTR_DMA_HT7,
 349         INSTR_DMA_HT8,
 350
 351         /* add dst src
 352          * dst += src
 353          * dst = HMEF, src = HMEFTI
 354          */
 355         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 356         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 357         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 358         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 359         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 360         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 361
 362         /* sub dst src
 363          * dst -= src
 364          * dst = HMEF, src = HMEFTI
 365          */
 366         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 367         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 368         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 369         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 370         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 371         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 372
 373         /* ckadd dst src
 374          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 375          * dst = H, src = {H, h.header}
 376          */
 377         INSTR_ALU_CKADD_FIELD,    /* src = H */
 378         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 379         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 380
 381         /* cksub dst src
 382          * dst = dst '- src
 383          * dst = H, src = H
 384          */
 385         INSTR_ALU_CKSUB_FIELD,
 386
 387         /* and dst src
 388          * dst &= src
 389          * dst = HMEF, src = HMEFTI
 390          */
 391         INSTR_ALU_AND,    /* dst = MEF, src = MEFT */
 392         INSTR_ALU_AND_MH, /* dst = MEF, src = H */
 393         INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
 394         INSTR_ALU_AND_HH, /* dst = H, src = H */
 395         INSTR_ALU_AND_I,  /* dst = HMEF, src = I */
 396
 397         /* or dst src
 398          * dst |= src
 399          * dst = HMEF, src = HMEFTI
 400          */
 401         INSTR_ALU_OR,    /* dst = MEF, src = MEFT */
 402         INSTR_ALU_OR_MH, /* dst = MEF, src = H */
 403         INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
 404         INSTR_ALU_OR_HH, /* dst = H, src = H */
 405         INSTR_ALU_OR_I,  /* dst = HMEF, src = I */
 406
 407         /* xor dst src
 408          * dst ^= src
 409          * dst = HMEF, src = HMEFTI
 410          */
 411         INSTR_ALU_XOR,    /* dst = MEF, src = MEFT */
 412         INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
 413         INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
 414         INSTR_ALU_XOR_HH, /* dst = H, src = H */
 415         INSTR_ALU_XOR_I,  /* dst = HMEF, src = I */
 416
 417         /* shl dst src
 418          * dst <<= src
 419          * dst = HMEF, src = HMEFTI
 420          */
 421         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 422         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 423         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 424         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 425         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 426         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 427
 428         /* shr dst src
 429          * dst >>= src
 430          * dst = HMEF, src = HMEFTI
 431          */
 432         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 433         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 434         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 435         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 436         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 437         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 438
 439         /* regprefetch REGARRAY index
 440          * prefetch REGARRAY[index]
 441          * index = HMEFTI
 442          */
 443         INSTR_REGPREFETCH_RH, /* index = H */
 444         INSTR_REGPREFETCH_RM, /* index = MEFT */
 445         INSTR_REGPREFETCH_RI, /* index = I */
 446
 447         /* regrd dst REGARRAY index
 448          * dst = REGARRAY[index]
 449          * dst = HMEF, index = HMEFTI
 450          */
 451         INSTR_REGRD_HRH, /* dst = H, index = H */
 452         INSTR_REGRD_HRM, /* dst = H, index = MEFT */
 453         INSTR_REGRD_HRI, /* dst = H, index = I */
 454         INSTR_REGRD_MRH, /* dst = MEF, index = H */
 455         INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
 456         INSTR_REGRD_MRI, /* dst = MEF, index = I */
 457
 458         /* regwr REGARRAY index src
 459          * REGARRAY[index] = src
 460          * index = HMEFTI, src = HMEFTI
 461          */
 462         INSTR_REGWR_RHH, /* index = H, src = H */
 463         INSTR_REGWR_RHM, /* index = H, src = MEFT */
 464         INSTR_REGWR_RHI, /* index = H, src = I */
 465         INSTR_REGWR_RMH, /* index = MEFT, src = H */
 466         INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
 467         INSTR_REGWR_RMI, /* index = MEFT, src = I */
 468         INSTR_REGWR_RIH, /* index = I, src = H */
 469         INSTR_REGWR_RIM, /* index = I, src = MEFT */
 470         INSTR_REGWR_RII, /* index = I, src = I */
 471
 472         /* regadd REGARRAY index src
 473          * REGARRAY[index] += src
 474          * index = HMEFTI, src = HMEFTI
 475          */
 476         INSTR_REGADD_RHH, /* index = H, src = H */
 477         INSTR_REGADD_RHM, /* index = H, src = MEFT */
 478         INSTR_REGADD_RHI, /* index = H, src = I */
 479         INSTR_REGADD_RMH, /* index = MEFT, src = H */
 480         INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
 481         INSTR_REGADD_RMI, /* index = MEFT, src = I */
 482         INSTR_REGADD_RIH, /* index = I, src = H */
 483         INSTR_REGADD_RIM, /* index = I, src = MEFT */
 484         INSTR_REGADD_RII, /* index = I, src = I */
 485
 486         /* metprefetch METARRAY index
 487          * prefetch METARRAY[index]
 488          * index = HMEFTI
 489          */
 490         INSTR_METPREFETCH_H, /* index = H */
 491         INSTR_METPREFETCH_M, /* index = MEFT */
 492         INSTR_METPREFETCH_I, /* index = I */
 493
 494         /* meter METARRAY index length color_in color_out
 495          * color_out = meter(METARRAY[index], length, color_in)
 496          * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
 497          */
 498         INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
 499         INSTR_METER_HHI, /* index = H, length = H, color_in = I */
 500         INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
 501         INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
 502         INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
 503         INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
 504         INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
 505         INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
 506         INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
 507         INSTR_METER_IHI, /* index = I, length = H, color_in = I */
 508         INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
 509         INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
 510
 511         /* table TABLE */
 512         INSTR_TABLE,
 513         INSTR_SELECTOR,
 514
 515         /* extern e.obj.func */
 516         INSTR_EXTERN_OBJ,
 517
 518         /* extern f.func */
 519         INSTR_EXTERN_FUNC,
 520
 521         /* jmp LABEL
 522          * Unconditional jump
 523          */
 524         INSTR_JMP,
 525
 526         /* jmpv LABEL h.header
 527          * Jump if header is valid
 528          */
 529         INSTR_JMP_VALID,
 530
 531         /* jmpnv LABEL h.header
 532          * Jump if header is invalid
 533          */
 534         INSTR_JMP_INVALID,
 535
 536         /* jmph LABEL
 537          * Jump if table lookup hit
 538          */
 539         INSTR_JMP_HIT,
 540
 541         /* jmpnh LABEL
 542          * Jump if table lookup miss
 543          */
 544         INSTR_JMP_MISS,
 545
 546         /* jmpa LABEL ACTION
 547          * Jump if action run
 548          */
 549         INSTR_JMP_ACTION_HIT,
 550
 551         /* jmpna LABEL ACTION
 552          * Jump if action not run
 553          */
 554         INSTR_JMP_ACTION_MISS,
 555
 556         /* jmpeq LABEL a b
 557          * Jump if a is equal to b
 558          * a = HMEFT, b = HMEFTI
 559          */
 560         INSTR_JMP_EQ,    /* a = MEFT, b = MEFT */
 561         INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
 562         INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
 563         INSTR_JMP_EQ_HH, /* a = H, b = H */
 564         INSTR_JMP_EQ_I,  /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 565
 566         /* jmpneq LABEL a b
 567          * Jump if a is not equal to b
 568          * a = HMEFT, b = HMEFTI
 569          */
 570         INSTR_JMP_NEQ,    /* a = MEFT, b = MEFT */
 571         INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
 572         INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
 573         INSTR_JMP_NEQ_HH, /* a = H, b = H */
 574         INSTR_JMP_NEQ_I,  /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 575
 576         /* jmplt LABEL a b
 577          * Jump if a is less than b
 578          * a = HMEFT, b = HMEFTI
 579          */
 580         INSTR_JMP_LT,    /* a = MEFT, b = MEFT */
 581         INSTR_JMP_LT_MH, /* a = MEFT, b = H */
 582         INSTR_JMP_LT_HM, /* a = H, b = MEFT */
 583         INSTR_JMP_LT_HH, /* a = H, b = H */
 584         INSTR_JMP_LT_MI, /* a = MEFT, b = I */
 585         INSTR_JMP_LT_HI, /* a = H, b = I */
 586
 587         /* jmpgt LABEL a b
 588          * Jump if a is greater than b
 589          * a = HMEFT, b = HMEFTI
 590          */
 591         INSTR_JMP_GT,    /* a = MEFT, b = MEFT */
 592         INSTR_JMP_GT_MH, /* a = MEFT, b = H */
 593         INSTR_JMP_GT_HM, /* a = H, b = MEFT */
 594         INSTR_JMP_GT_HH, /* a = H, b = H */
 595         INSTR_JMP_GT_MI, /* a = MEFT, b = I */
 596         INSTR_JMP_GT_HI, /* a = H, b = I */
 597
 598         /* return
 599          * Return from action
 600          */
 601         INSTR_RETURN,
 602 };
 603
 604 struct instr_operand {
 605         uint8_t struct_id;
 606         uint8_t n_bits;
 607         uint8_t offset;
 608         uint8_t pad;
 609 };
 610
 611 struct instr_io {
 612         struct {
 613                 union {
 614                         struct {
 615                                 uint8_t offset;
 616                                 uint8_t n_bits;
 617                                 uint8_t pad[2];
 618                         };
 619
 620                         uint32_t val;
 621                 };
 622         } io;
 623
 624         struct {
 625                 uint8_t header_id[8];
 626                 uint8_t struct_id[8];
 627                 uint8_t n_bytes[8];
 628         } hdr;
 629 };
 630
 631 struct instr_hdr_validity {
 632         uint8_t header_id;
 633 };
 634
 635 struct instr_table {
 636         uint8_t table_id;
 637 };
 638
 639 struct instr_extern_obj {
 640         uint8_t ext_obj_id;
 641         uint8_t func_id;
 642 };
 643
 644 struct instr_extern_func {
 645         uint8_t ext_func_id;
 646 };
 647
 648 struct instr_dst_src {
 649         struct instr_operand dst;
 650         union {
 651                 struct instr_operand src;
 652                 uint64_t src_val;
 653         };
 654 };
 655
 656 struct instr_regarray {
 657         uint8_t regarray_id;
 658         uint8_t pad[3];
 659
 660         union {
 661                 struct instr_operand idx;
 662                 uint32_t idx_val;
 663         };
 664
 665         union {
 666                 struct instr_operand dstsrc;
 667                 uint64_t dstsrc_val;
 668         };
 669 };
 670
 671 struct instr_meter {
 672         uint8_t metarray_id;
 673         uint8_t pad[3];
 674
 675         union {
 676                 struct instr_operand idx;
 677                 uint32_t idx_val;
 678         };
 679
 680         struct instr_operand length;
 681
 682         union {
 683                 struct instr_operand color_in;
 684                 uint32_t color_in_val;
 685         };
 686
 687         struct instr_operand color_out;
 688 };
 689
 690 struct instr_dma {
 691         struct {
 692                 uint8_t header_id[8];
 693                 uint8_t struct_id[8];
 694         } dst;
 695
 696         struct {
 697                 uint8_t offset[8];
 698         } src;
 699
 700         uint16_t n_bytes[8];
 701 };
 702
 703 struct instr_jmp {
 704         struct instruction *ip;
 705
 706         union {
 707                 struct instr_operand a;
 708                 uint8_t header_id;
 709                 uint8_t action_id;
 710         };
 711
 712         union {
 713                 struct instr_operand b;
 714                 uint64_t b_val;
 715         };
 716 };
 717
 718 struct instruction {
 719         enum instruction_type type;
 720         union {
 721                 struct instr_io io;
 722                 struct instr_hdr_validity valid;
 723                 struct instr_dst_src mov;
 724                 struct instr_regarray regarray;
 725                 struct instr_meter meter;
 726                 struct instr_dma dma;
 727                 struct instr_dst_src alu;
 728                 struct instr_table table;
 729                 struct instr_extern_obj ext_obj;
 730                 struct instr_extern_func ext_func;
 731                 struct instr_jmp jmp;
 732         };
 733 };
 734
 735 struct instruction_data {
 736         char label[RTE_SWX_NAME_SIZE];
 737         char jmp_label[RTE_SWX_NAME_SIZE];
 738         uint32_t n_users; /* user = jmp instruction to this instruction. */
 739         int invalid;
 740 };
 741
 742 /*
 743  * Action.
 744  */
 745 struct action {
 746         TAILQ_ENTRY(action) node;
 747         char name[RTE_SWX_NAME_SIZE];
 748         struct struct_type *st;
 749         int *args_endianness; /* 0 = Host Byte Order (HBO). */
 750         struct instruction *instructions;
 751         uint32_t n_instructions;
 752         uint32_t id;
 753 };
 754
 755 TAILQ_HEAD(action_tailq, action);
 756
 757 /*
 758  * Table.
 759  */
 760 struct table_type {
 761         TAILQ_ENTRY(table_type) node;
 762         char name[RTE_SWX_NAME_SIZE];
 763         enum rte_swx_table_match_type match_type;
 764         struct rte_swx_table_ops ops;
 765 };
 766
 767 TAILQ_HEAD(table_type_tailq, table_type);
 768
 769 struct match_field {
 770         enum rte_swx_table_match_type match_type;
 771         struct field *field;
 772 };
 773
 774 struct table {
 775         TAILQ_ENTRY(table) node;
 776         char name[RTE_SWX_NAME_SIZE];
 777         char args[RTE_SWX_NAME_SIZE];
 778         struct table_type *type; /* NULL when n_fields == 0. */
 779
 780         /* Match. */
 781         struct match_field *fields;
 782         uint32_t n_fields;
 783         struct header *header; /* Only valid when n_fields > 0. */
 784
 785         /* Action. */
 786         struct action **actions;
 787         struct action *default_action;
 788         uint8_t *default_action_data;
 789         uint32_t n_actions;
 790         int default_action_is_const;
 791         uint32_t action_data_size_max;
 792
 793         uint32_t size;
 794         uint32_t id;
 795 };
 796
 797 TAILQ_HEAD(table_tailq, table);
 798
 799 struct table_runtime {
 800         rte_swx_table_lookup_t func;
 801         void *mailbox;
 802         uint8_t **key;
 803 };
 804
 805 struct table_statistics {
 806         uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
 807         uint64_t *n_pkts_action;
 808 };
 809
 810 /*
 811  * Selector.
 812  */
 813 struct selector {
 814         TAILQ_ENTRY(selector) node;
 815         char name[RTE_SWX_NAME_SIZE];
 816
 817         struct field *group_id_field;
 818         struct field **selector_fields;
 819         uint32_t n_selector_fields;
 820         struct header *selector_header;
 821         struct field *member_id_field;
 822
 823         uint32_t n_groups_max;
 824         uint32_t n_members_per_group_max;
 825
 826         uint32_t id;
 827 };
 828
 829 TAILQ_HEAD(selector_tailq, selector);
 830
 831 struct selector_runtime {
 832         void *mailbox;
 833         uint8_t **group_id_buffer;
 834         uint8_t **selector_buffer;
 835         uint8_t **member_id_buffer;
 836 };
 837
 838 struct selector_statistics {
 839         uint64_t n_pkts;
 840 };
 841
 842 /*
 843  * Register array.
 844  */
 845 struct regarray {
 846         TAILQ_ENTRY(regarray) node;
 847         char name[RTE_SWX_NAME_SIZE];
 848         uint64_t init_val;
 849         uint32_t size;
 850         uint32_t id;
 851 };
 852
 853 TAILQ_HEAD(regarray_tailq, regarray);
 854
 855 struct regarray_runtime {
 856         uint64_t *regarray;
 857         uint32_t size_mask;
 858 };
 859
 860 /*
 861  * Meter array.
 862  */
 863 struct meter_profile {
 864         TAILQ_ENTRY(meter_profile) node;
 865         char name[RTE_SWX_NAME_SIZE];
 866         struct rte_meter_trtcm_params params;
 867         struct rte_meter_trtcm_profile profile;
 868         uint32_t n_users;
 869 };
 870
 871 TAILQ_HEAD(meter_profile_tailq, meter_profile);
 872
 873 struct metarray {
 874         TAILQ_ENTRY(metarray) node;
 875         char name[RTE_SWX_NAME_SIZE];
 876         uint32_t size;
 877         uint32_t id;
 878 };
 879
 880 TAILQ_HEAD(metarray_tailq, metarray);
 881
 882 struct meter {
 883         struct rte_meter_trtcm m;
 884         struct meter_profile *profile;
 885         enum rte_color color_mask;
 886         uint8_t pad[20];
 887
 888         uint64_t n_pkts[RTE_COLORS];
 889         uint64_t n_bytes[RTE_COLORS];
 890 };
 891
 892 struct metarray_runtime {
 893         struct meter *metarray;
 894         uint32_t size_mask;
 895 };
 896
 897 /*
 898  * Pipeline.
 899  */
 900 struct thread {
 901         /* Packet. */
 902         struct rte_swx_pkt pkt;
 903         uint8_t *ptr;
 904
 905         /* Structures. */
 906         uint8_t **structs;
 907
 908         /* Packet headers. */
 909         struct header_runtime *headers; /* Extracted or generated headers. */
 910         struct header_out_runtime *headers_out; /* Emitted headers. */
 911         uint8_t *header_storage;
 912         uint8_t *header_out_storage;
 913         uint64_t valid_headers;
 914         uint32_t n_headers_out;
 915
 916         /* Packet meta-data. */
 917         uint8_t *metadata;
 918
 919         /* Tables. */
 920         struct table_runtime *tables;
 921         struct selector_runtime *selectors;
 922         struct rte_swx_table_state *table_state;
 923         uint64_t action_id;
 924         int hit; /* 0 = Miss, 1 = Hit. */
 925
 926         /* Extern objects and functions. */
 927         struct extern_obj_runtime *extern_objs;
 928         struct extern_func_runtime *extern_funcs;
 929
 930         /* Instructions. */
 931         struct instruction *ip;
 932         struct instruction *ret;
 933 };
 934
 935 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 936 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 937 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 938
 939 #define HEADER_VALID(thread, header_id) \
 940         MASK64_BIT_GET((thread)->valid_headers, header_id)
 941
 942 #define ALU(thread, ip, operator)  \
 943 {                                                                              \
 944         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 945         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 946         uint64_t dst64 = *dst64_ptr;                                           \
 947         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 948         uint64_t dst = dst64 & dst64_mask;                                     \
 949                                                                                \
 950         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 951         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 952         uint64_t src64 = *src64_ptr;                                           \
 953         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 954         uint64_t src = src64 & src64_mask;                                     \
 955                                                                                \
 956         uint64_t result = dst operator src;                                    \
 957                                                                                \
 958         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 959 }
 960
 961 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 962
 963 #define ALU_MH(thread, ip, operator)  \
 964 {                                                                              \
 965         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 966         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 967         uint64_t dst64 = *dst64_ptr;                                           \
 968         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 969         uint64_t dst = dst64 & dst64_mask;                                     \
 970                                                                                \
 971         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 972         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 973         uint64_t src64 = *src64_ptr;                                           \
 974         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 975                                                                                \
 976         uint64_t result = dst operator src;                                    \
 977                                                                                \
 978         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 979 }
 980
 981 #define ALU_HM(thread, ip, operator)  \
 982 {                                                                              \
 983         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 984         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 985         uint64_t dst64 = *dst64_ptr;                                           \
 986         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 987         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 988                                                                                \
 989         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 990         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 991         uint64_t src64 = *src64_ptr;                                           \
 992         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 993         uint64_t src = src64 & src64_mask;                                     \
 994                                                                                \
 995         uint64_t result = dst operator src;                                    \
 996         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 997                                                                                \
 998         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 999 }
1000
1001 #define ALU_HM_FAST(thread, ip, operator)  \
1002 {                                                                                 \
1003         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];         \
1004         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];      \
1005         uint64_t dst64 = *dst64_ptr;                                              \
1006         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);          \
1007         uint64_t dst = dst64 & dst64_mask;                                        \
1008                                                                                   \
1009         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];         \
1010         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];      \
1011         uint64_t src64 = *src64_ptr;                                              \
1012         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);          \
1013         uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1014                                                                                   \
1015         uint64_t result = dst operator src;                                       \
1016                                                                                   \
1017         *dst64_ptr = (dst64 & ~dst64_mask) | result;                              \
1018 }
1019
1020 #define ALU_HH(thread, ip, operator)  \
1021 {                                                                              \
1022         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1023         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1024         uint64_t dst64 = *dst64_ptr;                                           \
1025         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1026         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
1027                                                                                \
1028         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
1029         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
1030         uint64_t src64 = *src64_ptr;                                           \
1031         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
1032                                                                                \
1033         uint64_t result = dst operator src;                                    \
1034         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1035                                                                                \
1036         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1037 }
1038
1039 #define ALU_HH_FAST(thread, ip, operator)  \
1040 {                                                                                             \
1041         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];                     \
1042         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];                  \
1043         uint64_t dst64 = *dst64_ptr;                                                          \
1044         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);                      \
1045         uint64_t dst = dst64 & dst64_mask;                                                    \
1046                                                                                               \
1047         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];                     \
1048         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];                  \
1049         uint64_t src64 = *src64_ptr;                                                          \
1050         uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1051                                                                                               \
1052         uint64_t result = dst operator src;                                                   \
1053                                                                                               \
1054         *dst64_ptr = (dst64 & ~dst64_mask) | result;                                          \
1055 }
1056
1057 #else
1058
1059 #define ALU_MH ALU
1060 #define ALU_HM ALU
1061 #define ALU_HM_FAST ALU
1062 #define ALU_HH ALU
1063 #define ALU_HH_FAST ALU
1064
1065 #endif
1066
1067 #define ALU_I(thread, ip, operator)  \
1068 {                                                                              \
1069         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1070         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1071         uint64_t dst64 = *dst64_ptr;                                           \
1072         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1073         uint64_t dst = dst64 & dst64_mask;                                     \
1074                                                                                \
1075         uint64_t src = (ip)->alu.src_val;                                      \
1076                                                                                \
1077         uint64_t result = dst operator src;                                    \
1078                                                                                \
1079         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
1080 }
1081
1082 #define ALU_MI ALU_I
1083
1084 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1085
1086 #define ALU_HI(thread, ip, operator)  \
1087 {                                                                              \
1088         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1089         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1090         uint64_t dst64 = *dst64_ptr;                                           \
1091         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1092         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
1093                                                                                \
1094         uint64_t src = (ip)->alu.src_val;                                      \
1095                                                                                \
1096         uint64_t result = dst operator src;                                    \
1097         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1098                                                                                \
1099         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1100 }
1101
1102 #else
1103
1104 #define ALU_HI ALU_I
1105
1106 #endif
1107
1108 #define MOV(thread, ip)  \
1109 {                                                                              \
1110         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1111         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1112         uint64_t dst64 = *dst64_ptr;                                           \
1113         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1114                                                                                \
1115         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1116         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1117         uint64_t src64 = *src64_ptr;                                           \
1118         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1119         uint64_t src = src64 & src64_mask;                                     \
1120                                                                                \
1121         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1122 }
1123
1124 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1125
1126 #define MOV_MH(thread, ip)  \
1127 {                                                                              \
1128         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1129         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1130         uint64_t dst64 = *dst64_ptr;                                           \
1131         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1132                                                                                \
1133         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1134         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1135         uint64_t src64 = *src64_ptr;                                           \
1136         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
1137                                                                                \
1138         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1139 }
1140
1141 #define MOV_HM(thread, ip)  \
1142 {                                                                              \
1143         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1144         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1145         uint64_t dst64 = *dst64_ptr;                                           \
1146         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1147                                                                                \
1148         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1149         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1150         uint64_t src64 = *src64_ptr;                                           \
1151         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1152         uint64_t src = src64 & src64_mask;                                     \
1153                                                                                \
1154         src = hton64(src) >> (64 - (ip)->mov.dst.n_bits);                      \
1155         *dst64_ptr = (dst64 & ~dst64_mask) | src;                              \
1156 }
1157
1158 #define MOV_HH(thread, ip)  \
1159 {                                                                              \
1160         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1161         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1162         uint64_t dst64 = *dst64_ptr;                                           \
1163         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1164                                                                                \
1165         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1166         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1167         uint64_t src64 = *src64_ptr;                                           \
1168                                                                                \
1169         uint64_t src = src64 << (64 - (ip)->mov.src.n_bits);                   \
1170         src = src >> (64 - (ip)->mov.dst.n_bits);                              \
1171         *dst64_ptr = (dst64 & ~dst64_mask) | src;                              \
1172 }
1173
1174 #else
1175
1176 #define MOV_MH MOV
1177 #define MOV_HM MOV
1178 #define MOV_HH MOV
1179
1180 #endif
1181
1182 #define MOV_I(thread, ip)  \
1183 {                                                                              \
1184         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1185         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1186         uint64_t dst64 = *dst64_ptr;                                           \
1187         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1188                                                                                \
1189         uint64_t src = (ip)->mov.src_val;                                      \
1190                                                                                \
1191         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1192 }
1193
1194 #define JMP_CMP(thread, ip, operator)  \
1195 {                                                                              \
1196         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1197         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1198         uint64_t a64 = *a64_ptr;                                               \
1199         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1200         uint64_t a = a64 & a64_mask;                                           \
1201                                                                                \
1202         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1203         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1204         uint64_t b64 = *b64_ptr;                                               \
1205         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1206         uint64_t b = b64 & b64_mask;                                           \
1207                                                                                \
1208         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1209 }
1210
1211 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1212
1213 #define JMP_CMP_MH(thread, ip, operator)  \
1214 {                                                                              \
1215         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1216         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1217         uint64_t a64 = *a64_ptr;                                               \
1218         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1219         uint64_t a = a64 & a64_mask;                                           \
1220                                                                                \
1221         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1222         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1223         uint64_t b64 = *b64_ptr;                                               \
1224         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1225                                                                                \
1226         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1227 }
1228
1229 #define JMP_CMP_HM(thread, ip, operator)  \
1230 {                                                                              \
1231         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1232         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1233         uint64_t a64 = *a64_ptr;                                               \
1234         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1235                                                                                \
1236         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1237         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1238         uint64_t b64 = *b64_ptr;                                               \
1239         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1240         uint64_t b = b64 & b64_mask;                                           \
1241                                                                                \
1242         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1243 }
1244
1245 #define JMP_CMP_HH(thread, ip, operator)  \
1246 {                                                                              \
1247         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1248         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1249         uint64_t a64 = *a64_ptr;                                               \
1250         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1251                                                                                \
1252         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1253         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1254         uint64_t b64 = *b64_ptr;                                               \
1255         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1256                                                                                \
1257         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1258 }
1259
1260 #define JMP_CMP_HH_FAST(thread, ip, operator)  \
1261 {                                                                              \
1262         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1263         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1264         uint64_t a64 = *a64_ptr;                                               \
1265         uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits);                         \
1266                                                                                \
1267         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1268         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1269         uint64_t b64 = *b64_ptr;                                               \
1270         uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits);                         \
1271                                                                                \
1272         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1273 }
1274
1275 #else
1276
1277 #define JMP_CMP_MH JMP_CMP
1278 #define JMP_CMP_HM JMP_CMP
1279 #define JMP_CMP_HH JMP_CMP
1280 #define JMP_CMP_HH_FAST JMP_CMP
1281
1282 #endif
1283
1284 #define JMP_CMP_I(thread, ip, operator)  \
1285 {                                                                              \
1286         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1287         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1288         uint64_t a64 = *a64_ptr;                                               \
1289         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1290         uint64_t a = a64 & a64_mask;                                           \
1291                                                                                \
1292         uint64_t b = (ip)->jmp.b_val;                                          \
1293                                                                                \
1294         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1295 }
1296
1297 #define JMP_CMP_MI JMP_CMP_I
1298
1299 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1300
1301 #define JMP_CMP_HI(thread, ip, operator)  \
1302 {                                                                              \
1303         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1304         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1305         uint64_t a64 = *a64_ptr;                                               \
1306         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1307                                                                                \
1308         uint64_t b = (ip)->jmp.b_val;                                          \
1309                                                                                \
1310         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1311 }
1312
1313 #else
1314
1315 #define JMP_CMP_HI JMP_CMP_I
1316
1317 #endif
1318
1319 #define METADATA_READ(thread, offset, n_bits)                                  \
1320 ({                                                                             \
1321         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1322         uint64_t m64 = *m64_ptr;                                               \
1323         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1324         (m64 & m64_mask);                                                      \
1325 })
1326
1327 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
1328 {                                                                              \
1329         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1330         uint64_t m64 = *m64_ptr;                                               \
1331         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1332                                                                                \
1333         uint64_t m_new = value;                                                \
1334                                                                                \
1335         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
1336 }
1337
1338 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1339 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1340 #endif
1341
1342 struct rte_swx_pipeline {
1343         struct struct_type_tailq struct_types;
1344         struct port_in_type_tailq port_in_types;
1345         struct port_in_tailq ports_in;
1346         struct port_out_type_tailq port_out_types;
1347         struct port_out_tailq ports_out;
1348         struct extern_type_tailq extern_types;
1349         struct extern_obj_tailq extern_objs;
1350         struct extern_func_tailq extern_funcs;
1351         struct header_tailq headers;
1352         struct struct_type *metadata_st;
1353         uint32_t metadata_struct_id;
1354         struct action_tailq actions;
1355         struct table_type_tailq table_types;
1356         struct table_tailq tables;
1357         struct selector_tailq selectors;
1358         struct regarray_tailq regarrays;
1359         struct meter_profile_tailq meter_profiles;
1360         struct metarray_tailq metarrays;
1361
1362         struct port_in_runtime *in;
1363         struct port_out_runtime *out;
1364         struct instruction **action_instructions;
1365         struct rte_swx_table_state *table_state;
1366         struct table_statistics *table_stats;
1367         struct selector_statistics *selector_stats;
1368         struct regarray_runtime *regarray_runtime;
1369         struct metarray_runtime *metarray_runtime;
1370         struct instruction *instructions;
1371         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1372
1373         uint32_t n_structs;
1374         uint32_t n_ports_in;
1375         uint32_t n_ports_out;
1376         uint32_t n_extern_objs;
1377         uint32_t n_extern_funcs;
1378         uint32_t n_actions;
1379         uint32_t n_tables;
1380         uint32_t n_selectors;
1381         uint32_t n_regarrays;
1382         uint32_t n_metarrays;
1383         uint32_t n_headers;
1384         uint32_t thread_id;
1385         uint32_t port_id;
1386         uint32_t n_instructions;
1387         int build_done;
1388         int numa_node;
1389 };
1390
1391 /*
1392  * Struct.
1393  */
1394 static struct struct_type *
1395 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1396 {
1397         struct struct_type *elem;
1398
1399         TAILQ_FOREACH(elem, &p->struct_types, node)
1400                 if (strcmp(elem->name, name) == 0)
1401                         return elem;
1402
1403         return NULL;
1404 }
1405
1406 static struct field *
1407 struct_type_field_find(struct struct_type *st, const char *name)
1408 {
1409         uint32_t i;
1410
1411         for (i = 0; i < st->n_fields; i++) {
1412                 struct field *f = &st->fields[i];
1413
1414                 if (strcmp(f->name, name) == 0)
1415                         return f;
1416         }
1417
1418         return NULL;
1419 }
1420
1421 int
1422 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1423                                       const char *name,
1424                                       struct rte_swx_field_params *fields,
1425                                       uint32_t n_fields,
1426                                       int last_field_has_variable_size)
1427 {
1428         struct struct_type *st;
1429         uint32_t i;
1430
1431         CHECK(p, EINVAL);
1432         CHECK_NAME(name, EINVAL);
1433         CHECK(fields, EINVAL);
1434         CHECK(n_fields, EINVAL);
1435
1436         for (i = 0; i < n_fields; i++) {
1437                 struct rte_swx_field_params *f = &fields[i];
1438                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
1439                 uint32_t j;
1440
1441                 CHECK_NAME(f->name, EINVAL);
1442                 CHECK(f->n_bits, EINVAL);
1443                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
1444                 CHECK((f->n_bits & 7) == 0, EINVAL);
1445
1446                 for (j = 0; j < i; j++) {
1447                         struct rte_swx_field_params *f_prev = &fields[j];
1448
1449                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
1450                 }
1451         }
1452
1453         CHECK(!struct_type_find(p, name), EEXIST);
1454
1455         /* Node allocation. */
1456         st = calloc(1, sizeof(struct struct_type));
1457         CHECK(st, ENOMEM);
1458
1459         st->fields = calloc(n_fields, sizeof(struct field));
1460         if (!st->fields) {
1461                 free(st);
1462                 CHECK(0, ENOMEM);
1463         }
1464
1465         /* Node initialization. */
1466         strcpy(st->name, name);
1467         for (i = 0; i < n_fields; i++) {
1468                 struct field *dst = &st->fields[i];
1469                 struct rte_swx_field_params *src = &fields[i];
1470                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
1471
1472                 strcpy(dst->name, src->name);
1473                 dst->n_bits = src->n_bits;
1474                 dst->offset = st->n_bits;
1475                 dst->var_size = var_size;
1476
1477                 st->n_bits += src->n_bits;
1478                 st->n_bits_min += var_size ? 0 : src->n_bits;
1479         }
1480         st->n_fields = n_fields;
1481         st->var_size = last_field_has_variable_size;
1482
1483         /* Node add to tailq. */
1484         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1485
1486         return 0;
1487 }
1488
1489 static int
1490 struct_build(struct rte_swx_pipeline *p)
1491 {
1492         uint32_t i;
1493
1494         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1495                 struct thread *t = &p->threads[i];
1496
1497                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1498                 CHECK(t->structs, ENOMEM);
1499         }
1500
1501         return 0;
1502 }
1503
1504 static void
1505 struct_build_free(struct rte_swx_pipeline *p)
1506 {
1507         uint32_t i;
1508
1509         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1510                 struct thread *t = &p->threads[i];
1511
1512                 free(t->structs);
1513                 t->structs = NULL;
1514         }
1515 }
1516
1517 static void
1518 struct_free(struct rte_swx_pipeline *p)
1519 {
1520         struct_build_free(p);
1521
1522         /* Struct types. */
1523         for ( ; ; ) {
1524                 struct struct_type *elem;
1525
1526                 elem = TAILQ_FIRST(&p->struct_types);
1527                 if (!elem)
1528                         break;
1529
1530                 TAILQ_REMOVE(&p->struct_types, elem, node);
1531                 free(elem->fields);
1532                 free(elem);
1533         }
1534 }
1535
1536 /*
1537  * Input port.
1538  */
1539 static struct port_in_type *
1540 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1541 {
1542         struct port_in_type *elem;
1543
1544         if (!name)
1545                 return NULL;
1546
1547         TAILQ_FOREACH(elem, &p->port_in_types, node)
1548                 if (strcmp(elem->name, name) == 0)
1549                         return elem;
1550
1551         return NULL;
1552 }
1553
1554 int
1555 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1556                                        const char *name,
1557                                        struct rte_swx_port_in_ops *ops)
1558 {
1559         struct port_in_type *elem;
1560
1561         CHECK(p, EINVAL);
1562         CHECK_NAME(name, EINVAL);
1563         CHECK(ops, EINVAL);
1564         CHECK(ops->create, EINVAL);
1565         CHECK(ops->free, EINVAL);
1566         CHECK(ops->pkt_rx, EINVAL);
1567         CHECK(ops->stats_read, EINVAL);
1568
1569         CHECK(!port_in_type_find(p, name), EEXIST);
1570
1571         /* Node allocation. */
1572         elem = calloc(1, sizeof(struct port_in_type));
1573         CHECK(elem, ENOMEM);
1574
1575         /* Node initialization. */
1576         strcpy(elem->name, name);
1577         memcpy(&elem->ops, ops, sizeof(*ops));
1578
1579         /* Node add to tailq. */
1580         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1581
1582         return 0;
1583 }
1584
1585 static struct port_in *
1586 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1587 {
1588         struct port_in *port;
1589
1590         TAILQ_FOREACH(port, &p->ports_in, node)
1591                 if (port->id == port_id)
1592                         return port;
1593
1594         return NULL;
1595 }
1596
1597 int
1598 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1599                                 uint32_t port_id,
1600                                 const char *port_type_name,
1601                                 void *args)
1602 {
1603         struct port_in_type *type = NULL;
1604         struct port_in *port = NULL;
1605         void *obj = NULL;
1606
1607         CHECK(p, EINVAL);
1608
1609         CHECK(!port_in_find(p, port_id), EINVAL);
1610
1611         CHECK_NAME(port_type_name, EINVAL);
1612         type = port_in_type_find(p, port_type_name);
1613         CHECK(type, EINVAL);
1614
1615         obj = type->ops.create(args);
1616         CHECK(obj, ENODEV);
1617
1618         /* Node allocation. */
1619         port = calloc(1, sizeof(struct port_in));
1620         CHECK(port, ENOMEM);
1621
1622         /* Node initialization. */
1623         port->type = type;
1624         port->obj = obj;
1625         port->id = port_id;
1626
1627         /* Node add to tailq. */
1628         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1629         if (p->n_ports_in < port_id + 1)
1630                 p->n_ports_in = port_id + 1;
1631
1632         return 0;
1633 }
1634
1635 static int
1636 port_in_build(struct rte_swx_pipeline *p)
1637 {
1638         struct port_in *port;
1639         uint32_t i;
1640
1641         CHECK(p->n_ports_in, EINVAL);
1642         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1643
1644         for (i = 0; i < p->n_ports_in; i++)
1645                 CHECK(port_in_find(p, i), EINVAL);
1646
1647         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1648         CHECK(p->in, ENOMEM);
1649
1650         TAILQ_FOREACH(port, &p->ports_in, node) {
1651                 struct port_in_runtime *in = &p->in[port->id];
1652
1653                 in->pkt_rx = port->type->ops.pkt_rx;
1654                 in->obj = port->obj;
1655         }
1656
1657         return 0;
1658 }
1659
1660 static void
1661 port_in_build_free(struct rte_swx_pipeline *p)
1662 {
1663         free(p->in);
1664         p->in = NULL;
1665 }
1666
1667 static void
1668 port_in_free(struct rte_swx_pipeline *p)
1669 {
1670         port_in_build_free(p);
1671
1672         /* Input ports. */
1673         for ( ; ; ) {
1674                 struct port_in *port;
1675
1676                 port = TAILQ_FIRST(&p->ports_in);
1677                 if (!port)
1678                         break;
1679
1680                 TAILQ_REMOVE(&p->ports_in, port, node);
1681                 port->type->ops.free(port->obj);
1682                 free(port);
1683         }
1684
1685         /* Input port types. */
1686         for ( ; ; ) {
1687                 struct port_in_type *elem;
1688
1689                 elem = TAILQ_FIRST(&p->port_in_types);
1690                 if (!elem)
1691                         break;
1692
1693                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1694                 free(elem);
1695         }
1696 }
1697
1698 /*
1699  * Output port.
1700  */
1701 static struct port_out_type *
1702 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1703 {
1704         struct port_out_type *elem;
1705
1706         if (!name)
1707                 return NULL;
1708
1709         TAILQ_FOREACH(elem, &p->port_out_types, node)
1710                 if (!strcmp(elem->name, name))
1711                         return elem;
1712
1713         return NULL;
1714 }
1715
1716 int
1717 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1718                                         const char *name,
1719                                         struct rte_swx_port_out_ops *ops)
1720 {
1721         struct port_out_type *elem;
1722
1723         CHECK(p, EINVAL);
1724         CHECK_NAME(name, EINVAL);
1725         CHECK(ops, EINVAL);
1726         CHECK(ops->create, EINVAL);
1727         CHECK(ops->free, EINVAL);
1728         CHECK(ops->pkt_tx, EINVAL);
1729         CHECK(ops->stats_read, EINVAL);
1730
1731         CHECK(!port_out_type_find(p, name), EEXIST);
1732
1733         /* Node allocation. */
1734         elem = calloc(1, sizeof(struct port_out_type));
1735         CHECK(elem, ENOMEM);
1736
1737         /* Node initialization. */
1738         strcpy(elem->name, name);
1739         memcpy(&elem->ops, ops, sizeof(*ops));
1740
1741         /* Node add to tailq. */
1742         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1743
1744         return 0;
1745 }
1746
1747 static struct port_out *
1748 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1749 {
1750         struct port_out *port;
1751
1752         TAILQ_FOREACH(port, &p->ports_out, node)
1753                 if (port->id == port_id)
1754                         return port;
1755
1756         return NULL;
1757 }
1758
1759 int
1760 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1761                                  uint32_t port_id,
1762                                  const char *port_type_name,
1763                                  void *args)
1764 {
1765         struct port_out_type *type = NULL;
1766         struct port_out *port = NULL;
1767         void *obj = NULL;
1768
1769         CHECK(p, EINVAL);
1770
1771         CHECK(!port_out_find(p, port_id), EINVAL);
1772
1773         CHECK_NAME(port_type_name, EINVAL);
1774         type = port_out_type_find(p, port_type_name);
1775         CHECK(type, EINVAL);
1776
1777         obj = type->ops.create(args);
1778         CHECK(obj, ENODEV);
1779
1780         /* Node allocation. */
1781         port = calloc(1, sizeof(struct port_out));
1782         CHECK(port, ENOMEM);
1783
1784         /* Node initialization. */
1785         port->type = type;
1786         port->obj = obj;
1787         port->id = port_id;
1788
1789         /* Node add to tailq. */
1790         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1791         if (p->n_ports_out < port_id + 1)
1792                 p->n_ports_out = port_id + 1;
1793
1794         return 0;
1795 }
1796
1797 static int
1798 port_out_build(struct rte_swx_pipeline *p)
1799 {
1800         struct port_out *port;
1801         uint32_t i;
1802
1803         CHECK(p->n_ports_out, EINVAL);
1804
1805         for (i = 0; i < p->n_ports_out; i++)
1806                 CHECK(port_out_find(p, i), EINVAL);
1807
1808         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1809         CHECK(p->out, ENOMEM);
1810
1811         TAILQ_FOREACH(port, &p->ports_out, node) {
1812                 struct port_out_runtime *out = &p->out[port->id];
1813
1814                 out->pkt_tx = port->type->ops.pkt_tx;
1815                 out->flush = port->type->ops.flush;
1816                 out->obj = port->obj;
1817         }
1818
1819         return 0;
1820 }
1821
1822 static void
1823 port_out_build_free(struct rte_swx_pipeline *p)
1824 {
1825         free(p->out);
1826         p->out = NULL;
1827 }
1828
1829 static void
1830 port_out_free(struct rte_swx_pipeline *p)
1831 {
1832         port_out_build_free(p);
1833
1834         /* Output ports. */
1835         for ( ; ; ) {
1836                 struct port_out *port;
1837
1838                 port = TAILQ_FIRST(&p->ports_out);
1839                 if (!port)
1840                         break;
1841
1842                 TAILQ_REMOVE(&p->ports_out, port, node);
1843                 port->type->ops.free(port->obj);
1844                 free(port);
1845         }
1846
1847         /* Output port types. */
1848         for ( ; ; ) {
1849                 struct port_out_type *elem;
1850
1851                 elem = TAILQ_FIRST(&p->port_out_types);
1852                 if (!elem)
1853                         break;
1854
1855                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1856                 free(elem);
1857         }
1858 }
1859
1860 /*
1861  * Extern object.
1862  */
1863 static struct extern_type *
1864 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1865 {
1866         struct extern_type *elem;
1867
1868         TAILQ_FOREACH(elem, &p->extern_types, node)
1869                 if (strcmp(elem->name, name) == 0)
1870                         return elem;
1871
1872         return NULL;
1873 }
1874
1875 static struct extern_type_member_func *
1876 extern_type_member_func_find(struct extern_type *type, const char *name)
1877 {
1878         struct extern_type_member_func *elem;
1879
1880         TAILQ_FOREACH(elem, &type->funcs, node)
1881                 if (strcmp(elem->name, name) == 0)
1882                         return elem;
1883
1884         return NULL;
1885 }
1886
1887 static struct extern_obj *
1888 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1889 {
1890         struct extern_obj *elem;
1891
1892         TAILQ_FOREACH(elem, &p->extern_objs, node)
1893                 if (strcmp(elem->name, name) == 0)
1894                         return elem;
1895
1896         return NULL;
1897 }
1898
1899 static struct extern_type_member_func *
1900 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1901                              const char *name,
1902                              struct extern_obj **obj)
1903 {
1904         struct extern_obj *object;
1905         struct extern_type_member_func *func;
1906         char *object_name, *func_name;
1907
1908         if (name[0] != 'e' || name[1] != '.')
1909                 return NULL;
1910
1911         object_name = strdup(&name[2]);
1912         if (!object_name)
1913                 return NULL;
1914
1915         func_name = strchr(object_name, '.');
1916         if (!func_name) {
1917                 free(object_name);
1918                 return NULL;
1919         }
1920
1921         *func_name = 0;
1922         func_name++;
1923
1924         object = extern_obj_find(p, object_name);
1925         if (!object) {
1926                 free(object_name);
1927                 return NULL;
1928         }
1929
1930         func = extern_type_member_func_find(object->type, func_name);
1931         if (!func) {
1932                 free(object_name);
1933                 return NULL;
1934         }
1935
1936         if (obj)
1937                 *obj = object;
1938
1939         free(object_name);
1940         return func;
1941 }
1942
1943 static struct field *
1944 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1945                                const char *name,
1946                                struct extern_obj **object)
1947 {
1948         struct extern_obj *obj;
1949         struct field *f;
1950         char *obj_name, *field_name;
1951
1952         if ((name[0] != 'e') || (name[1] != '.'))
1953                 return NULL;
1954
1955         obj_name = strdup(&name[2]);
1956         if (!obj_name)
1957                 return NULL;
1958
1959         field_name = strchr(obj_name, '.');
1960         if (!field_name) {
1961                 free(obj_name);
1962                 return NULL;
1963         }
1964
1965         *field_name = 0;
1966         field_name++;
1967
1968         obj = extern_obj_find(p, obj_name);
1969         if (!obj) {
1970                 free(obj_name);
1971                 return NULL;
1972         }
1973
1974         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1975         if (!f) {
1976                 free(obj_name);
1977                 return NULL;
1978         }
1979
1980         if (object)
1981                 *object = obj;
1982
1983         free(obj_name);
1984         return f;
1985 }
1986
1987 int
1988 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1989         const char *name,
1990         const char *mailbox_struct_type_name,
1991         rte_swx_extern_type_constructor_t constructor,
1992         rte_swx_extern_type_destructor_t destructor)
1993 {
1994         struct extern_type *elem;
1995         struct struct_type *mailbox_struct_type;
1996
1997         CHECK(p, EINVAL);
1998
1999         CHECK_NAME(name, EINVAL);
2000         CHECK(!extern_type_find(p, name), EEXIST);
2001
2002         CHECK_NAME(mailbox_struct_type_name, EINVAL);
2003         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2004         CHECK(mailbox_struct_type, EINVAL);
2005         CHECK(!mailbox_struct_type->var_size, EINVAL);
2006
2007         CHECK(constructor, EINVAL);
2008         CHECK(destructor, EINVAL);
2009
2010         /* Node allocation. */
2011         elem = calloc(1, sizeof(struct extern_type));
2012         CHECK(elem, ENOMEM);
2013
2014         /* Node initialization. */
2015         strcpy(elem->name, name);
2016         elem->mailbox_struct_type = mailbox_struct_type;
2017         elem->constructor = constructor;
2018         elem->destructor = destructor;
2019         TAILQ_INIT(&elem->funcs);
2020
2021         /* Node add to tailq. */
2022         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
2023
2024         return 0;
2025 }
2026
2027 int
2028 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
2029         const char *extern_type_name,
2030         const char *name,
2031         rte_swx_extern_type_member_func_t member_func)
2032 {
2033         struct extern_type *type;
2034         struct extern_type_member_func *type_member;
2035
2036         CHECK(p, EINVAL);
2037
2038         CHECK_NAME(extern_type_name, EINVAL);
2039         type = extern_type_find(p, extern_type_name);
2040         CHECK(type, EINVAL);
2041         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
2042
2043         CHECK_NAME(name, EINVAL);
2044         CHECK(!extern_type_member_func_find(type, name), EEXIST);
2045
2046         CHECK(member_func, EINVAL);
2047
2048         /* Node allocation. */
2049         type_member = calloc(1, sizeof(struct extern_type_member_func));
2050         CHECK(type_member, ENOMEM);
2051
2052         /* Node initialization. */
2053         strcpy(type_member->name, name);
2054         type_member->func = member_func;
2055         type_member->id = type->n_funcs;
2056
2057         /* Node add to tailq. */
2058         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
2059         type->n_funcs++;
2060
2061         return 0;
2062 }
2063
2064 int
2065 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
2066                                       const char *extern_type_name,
2067                                       const char *name,
2068                                       const char *args)
2069 {
2070         struct extern_type *type;
2071         struct extern_obj *obj;
2072         void *obj_handle;
2073
2074         CHECK(p, EINVAL);
2075
2076         CHECK_NAME(extern_type_name, EINVAL);
2077         type = extern_type_find(p, extern_type_name);
2078         CHECK(type, EINVAL);
2079
2080         CHECK_NAME(name, EINVAL);
2081         CHECK(!extern_obj_find(p, name), EEXIST);
2082
2083         /* Node allocation. */
2084         obj = calloc(1, sizeof(struct extern_obj));
2085         CHECK(obj, ENOMEM);
2086
2087         /* Object construction. */
2088         obj_handle = type->constructor(args);
2089         if (!obj_handle) {
2090                 free(obj);
2091                 CHECK(0, ENODEV);
2092         }
2093
2094         /* Node initialization. */
2095         strcpy(obj->name, name);
2096         obj->type = type;
2097         obj->obj = obj_handle;
2098         obj->struct_id = p->n_structs;
2099         obj->id = p->n_extern_objs;
2100
2101         /* Node add to tailq. */
2102         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
2103         p->n_extern_objs++;
2104         p->n_structs++;
2105
2106         return 0;
2107 }
2108
2109 static int
2110 extern_obj_build(struct rte_swx_pipeline *p)
2111 {
2112         uint32_t i;
2113
2114         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2115                 struct thread *t = &p->threads[i];
2116                 struct extern_obj *obj;
2117
2118                 t->extern_objs = calloc(p->n_extern_objs,
2119                                         sizeof(struct extern_obj_runtime));
2120                 CHECK(t->extern_objs, ENOMEM);
2121
2122                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
2123                         struct extern_obj_runtime *r =
2124                                 &t->extern_objs[obj->id];
2125                         struct extern_type_member_func *func;
2126                         uint32_t mailbox_size =
2127                                 obj->type->mailbox_struct_type->n_bits / 8;
2128
2129                         r->obj = obj->obj;
2130
2131                         r->mailbox = calloc(1, mailbox_size);
2132                         CHECK(r->mailbox, ENOMEM);
2133
2134                         TAILQ_FOREACH(func, &obj->type->funcs, node)
2135                                 r->funcs[func->id] = func->func;
2136
2137                         t->structs[obj->struct_id] = r->mailbox;
2138                 }
2139         }
2140
2141         return 0;
2142 }
2143
2144 static void
2145 extern_obj_build_free(struct rte_swx_pipeline *p)
2146 {
2147         uint32_t i;
2148
2149         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2150                 struct thread *t = &p->threads[i];
2151                 uint32_t j;
2152
2153                 if (!t->extern_objs)
2154                         continue;
2155
2156                 for (j = 0; j < p->n_extern_objs; j++) {
2157                         struct extern_obj_runtime *r = &t->extern_objs[j];
2158
2159                         free(r->mailbox);
2160                 }
2161
2162                 free(t->extern_objs);
2163                 t->extern_objs = NULL;
2164         }
2165 }
2166
2167 static void
2168 extern_obj_free(struct rte_swx_pipeline *p)
2169 {
2170         extern_obj_build_free(p);
2171
2172         /* Extern objects. */
2173         for ( ; ; ) {
2174                 struct extern_obj *elem;
2175
2176                 elem = TAILQ_FIRST(&p->extern_objs);
2177                 if (!elem)
2178                         break;
2179
2180                 TAILQ_REMOVE(&p->extern_objs, elem, node);
2181                 if (elem->obj)
2182                         elem->type->destructor(elem->obj);
2183                 free(elem);
2184         }
2185
2186         /* Extern types. */
2187         for ( ; ; ) {
2188                 struct extern_type *elem;
2189
2190                 elem = TAILQ_FIRST(&p->extern_types);
2191                 if (!elem)
2192                         break;
2193
2194                 TAILQ_REMOVE(&p->extern_types, elem, node);
2195
2196                 for ( ; ; ) {
2197                         struct extern_type_member_func *func;
2198
2199                         func = TAILQ_FIRST(&elem->funcs);
2200                         if (!func)
2201                                 break;
2202
2203                         TAILQ_REMOVE(&elem->funcs, func, node);
2204                         free(func);
2205                 }
2206
2207                 free(elem);
2208         }
2209 }
2210
2211 /*
2212  * Extern function.
2213  */
2214 static struct extern_func *
2215 extern_func_find(struct rte_swx_pipeline *p, const char *name)
2216 {
2217         struct extern_func *elem;
2218
2219         TAILQ_FOREACH(elem, &p->extern_funcs, node)
2220                 if (strcmp(elem->name, name) == 0)
2221                         return elem;
2222
2223         return NULL;
2224 }
2225
2226 static struct extern_func *
2227 extern_func_parse(struct rte_swx_pipeline *p,
2228                   const char *name)
2229 {
2230         if (name[0] != 'f' || name[1] != '.')
2231                 return NULL;
2232
2233         return extern_func_find(p, &name[2]);
2234 }
2235
2236 static struct field *
2237 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
2238                                 const char *name,
2239                                 struct extern_func **function)
2240 {
2241         struct extern_func *func;
2242         struct field *f;
2243         char *func_name, *field_name;
2244
2245         if ((name[0] != 'f') || (name[1] != '.'))
2246                 return NULL;
2247
2248         func_name = strdup(&name[2]);
2249         if (!func_name)
2250                 return NULL;
2251
2252         field_name = strchr(func_name, '.');
2253         if (!field_name) {
2254                 free(func_name);
2255                 return NULL;
2256         }
2257
2258         *field_name = 0;
2259         field_name++;
2260
2261         func = extern_func_find(p, func_name);
2262         if (!func) {
2263                 free(func_name);
2264                 return NULL;
2265         }
2266
2267         f = struct_type_field_find(func->mailbox_struct_type, field_name);
2268         if (!f) {
2269                 free(func_name);
2270                 return NULL;
2271         }
2272
2273         if (function)
2274                 *function = func;
2275
2276         free(func_name);
2277         return f;
2278 }
2279
2280 int
2281 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
2282                                       const char *name,
2283                                       const char *mailbox_struct_type_name,
2284                                       rte_swx_extern_func_t func)
2285 {
2286         struct extern_func *f;
2287         struct struct_type *mailbox_struct_type;
2288
2289         CHECK(p, EINVAL);
2290
2291         CHECK_NAME(name, EINVAL);
2292         CHECK(!extern_func_find(p, name), EEXIST);
2293
2294         CHECK_NAME(mailbox_struct_type_name, EINVAL);
2295         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2296         CHECK(mailbox_struct_type, EINVAL);
2297         CHECK(!mailbox_struct_type->var_size, EINVAL);
2298
2299         CHECK(func, EINVAL);
2300
2301         /* Node allocation. */
2302         f = calloc(1, sizeof(struct extern_func));
2303         CHECK(func, ENOMEM);
2304
2305         /* Node initialization. */
2306         strcpy(f->name, name);
2307         f->mailbox_struct_type = mailbox_struct_type;
2308         f->func = func;
2309         f->struct_id = p->n_structs;
2310         f->id = p->n_extern_funcs;
2311
2312         /* Node add to tailq. */
2313         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
2314         p->n_extern_funcs++;
2315         p->n_structs++;
2316
2317         return 0;
2318 }
2319
2320 static int
2321 extern_func_build(struct rte_swx_pipeline *p)
2322 {
2323         uint32_t i;
2324
2325         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2326                 struct thread *t = &p->threads[i];
2327                 struct extern_func *func;
2328
2329                 /* Memory allocation. */
2330                 t->extern_funcs = calloc(p->n_extern_funcs,
2331                                          sizeof(struct extern_func_runtime));
2332                 CHECK(t->extern_funcs, ENOMEM);
2333
2334                 /* Extern function. */
2335                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
2336                         struct extern_func_runtime *r =
2337                                 &t->extern_funcs[func->id];
2338                         uint32_t mailbox_size =
2339                                 func->mailbox_struct_type->n_bits / 8;
2340
2341                         r->func = func->func;
2342
2343                         r->mailbox = calloc(1, mailbox_size);
2344                         CHECK(r->mailbox, ENOMEM);
2345
2346                         t->structs[func->struct_id] = r->mailbox;
2347                 }
2348         }
2349
2350         return 0;
2351 }
2352
2353 static void
2354 extern_func_build_free(struct rte_swx_pipeline *p)
2355 {
2356         uint32_t i;
2357
2358         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2359                 struct thread *t = &p->threads[i];
2360                 uint32_t j;
2361
2362                 if (!t->extern_funcs)
2363                         continue;
2364
2365                 for (j = 0; j < p->n_extern_funcs; j++) {
2366                         struct extern_func_runtime *r = &t->extern_funcs[j];
2367
2368                         free(r->mailbox);
2369                 }
2370
2371                 free(t->extern_funcs);
2372                 t->extern_funcs = NULL;
2373         }
2374 }
2375
2376 static void
2377 extern_func_free(struct rte_swx_pipeline *p)
2378 {
2379         extern_func_build_free(p);
2380
2381         for ( ; ; ) {
2382                 struct extern_func *elem;
2383
2384                 elem = TAILQ_FIRST(&p->extern_funcs);
2385                 if (!elem)
2386                         break;
2387
2388                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
2389                 free(elem);
2390         }
2391 }
2392
2393 /*
2394  * Header.
2395  */
2396 static struct header *
2397 header_find(struct rte_swx_pipeline *p, const char *name)
2398 {
2399         struct header *elem;
2400
2401         TAILQ_FOREACH(elem, &p->headers, node)
2402                 if (strcmp(elem->name, name) == 0)
2403                         return elem;
2404
2405         return NULL;
2406 }
2407
2408 static struct header *
2409 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
2410 {
2411         struct header *elem;
2412
2413         TAILQ_FOREACH(elem, &p->headers, node)
2414                 if (elem->struct_id == struct_id)
2415                         return elem;
2416
2417         return NULL;
2418 }
2419
2420 static struct header *
2421 header_parse(struct rte_swx_pipeline *p,
2422              const char *name)
2423 {
2424         if (name[0] != 'h' || name[1] != '.')
2425                 return NULL;
2426
2427         return header_find(p, &name[2]);
2428 }
2429
2430 static struct field *
2431 header_field_parse(struct rte_swx_pipeline *p,
2432                    const char *name,
2433                    struct header **header)
2434 {
2435         struct header *h;
2436         struct field *f;
2437         char *header_name, *field_name;
2438
2439         if ((name[0] != 'h') || (name[1] != '.'))
2440                 return NULL;
2441
2442         header_name = strdup(&name[2]);
2443         if (!header_name)
2444                 return NULL;
2445
2446         field_name = strchr(header_name, '.');
2447         if (!field_name) {
2448                 free(header_name);
2449                 return NULL;
2450         }
2451
2452         *field_name = 0;
2453         field_name++;
2454
2455         h = header_find(p, header_name);
2456         if (!h) {
2457                 free(header_name);
2458                 return NULL;
2459         }
2460
2461         f = struct_type_field_find(h->st, field_name);
2462         if (!f) {
2463                 free(header_name);
2464                 return NULL;
2465         }
2466
2467         if (header)
2468                 *header = h;
2469
2470         free(header_name);
2471         return f;
2472 }
2473
2474 int
2475 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2476                                         const char *name,
2477                                         const char *struct_type_name)
2478 {
2479         struct struct_type *st;
2480         struct header *h;
2481         size_t n_headers_max;
2482
2483         CHECK(p, EINVAL);
2484         CHECK_NAME(name, EINVAL);
2485         CHECK_NAME(struct_type_name, EINVAL);
2486
2487         CHECK(!header_find(p, name), EEXIST);
2488
2489         st = struct_type_find(p, struct_type_name);
2490         CHECK(st, EINVAL);
2491
2492         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2493         CHECK(p->n_headers < n_headers_max, ENOSPC);
2494
2495         /* Node allocation. */
2496         h = calloc(1, sizeof(struct header));
2497         CHECK(h, ENOMEM);
2498
2499         /* Node initialization. */
2500         strcpy(h->name, name);
2501         h->st = st;
2502         h->struct_id = p->n_structs;
2503         h->id = p->n_headers;
2504
2505         /* Node add to tailq. */
2506         TAILQ_INSERT_TAIL(&p->headers, h, node);
2507         p->n_headers++;
2508         p->n_structs++;
2509
2510         return 0;
2511 }
2512
2513 static int
2514 header_build(struct rte_swx_pipeline *p)
2515 {
2516         struct header *h;
2517         uint32_t n_bytes = 0, i;
2518
2519         TAILQ_FOREACH(h, &p->headers, node) {
2520                 n_bytes += h->st->n_bits / 8;
2521         }
2522
2523         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2524                 struct thread *t = &p->threads[i];
2525                 uint32_t offset = 0;
2526
2527                 t->headers = calloc(p->n_headers,
2528                                     sizeof(struct header_runtime));
2529                 CHECK(t->headers, ENOMEM);
2530
2531                 t->headers_out = calloc(p->n_headers,
2532                                         sizeof(struct header_out_runtime));
2533                 CHECK(t->headers_out, ENOMEM);
2534
2535                 t->header_storage = calloc(1, n_bytes);
2536                 CHECK(t->header_storage, ENOMEM);
2537
2538                 t->header_out_storage = calloc(1, n_bytes);
2539                 CHECK(t->header_out_storage, ENOMEM);
2540
2541                 TAILQ_FOREACH(h, &p->headers, node) {
2542                         uint8_t *header_storage;
2543                         uint32_t n_bytes =  h->st->n_bits / 8;
2544
2545                         header_storage = &t->header_storage[offset];
2546                         offset += n_bytes;
2547
2548                         t->headers[h->id].ptr0 = header_storage;
2549                         t->headers[h->id].n_bytes = n_bytes;
2550
2551                         t->structs[h->struct_id] = header_storage;
2552                 }
2553         }
2554
2555         return 0;
2556 }
2557
2558 static void
2559 header_build_free(struct rte_swx_pipeline *p)
2560 {
2561         uint32_t i;
2562
2563         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2564                 struct thread *t = &p->threads[i];
2565
2566                 free(t->headers_out);
2567                 t->headers_out = NULL;
2568
2569                 free(t->headers);
2570                 t->headers = NULL;
2571
2572                 free(t->header_out_storage);
2573                 t->header_out_storage = NULL;
2574
2575                 free(t->header_storage);
2576                 t->header_storage = NULL;
2577         }
2578 }
2579
2580 static void
2581 header_free(struct rte_swx_pipeline *p)
2582 {
2583         header_build_free(p);
2584
2585         for ( ; ; ) {
2586                 struct header *elem;
2587
2588                 elem = TAILQ_FIRST(&p->headers);
2589                 if (!elem)
2590                         break;
2591
2592                 TAILQ_REMOVE(&p->headers, elem, node);
2593                 free(elem);
2594         }
2595 }
2596
2597 /*
2598  * Meta-data.
2599  */
2600 static struct field *
2601 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2602 {
2603         if (!p->metadata_st)
2604                 return NULL;
2605
2606         if (name[0] != 'm' || name[1] != '.')
2607                 return NULL;
2608
2609         return struct_type_field_find(p->metadata_st, &name[2]);
2610 }
2611
2612 int
2613 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2614                                           const char *struct_type_name)
2615 {
2616         struct struct_type *st = NULL;
2617
2618         CHECK(p, EINVAL);
2619
2620         CHECK_NAME(struct_type_name, EINVAL);
2621         st  = struct_type_find(p, struct_type_name);
2622         CHECK(st, EINVAL);
2623         CHECK(!st->var_size, EINVAL);
2624         CHECK(!p->metadata_st, EINVAL);
2625
2626         p->metadata_st = st;
2627         p->metadata_struct_id = p->n_structs;
2628
2629         p->n_structs++;
2630
2631         return 0;
2632 }
2633
2634 static int
2635 metadata_build(struct rte_swx_pipeline *p)
2636 {
2637         uint32_t n_bytes = p->metadata_st->n_bits / 8;
2638         uint32_t i;
2639
2640         /* Thread-level initialization. */
2641         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2642                 struct thread *t = &p->threads[i];
2643                 uint8_t *metadata;
2644
2645                 metadata = calloc(1, n_bytes);
2646                 CHECK(metadata, ENOMEM);
2647
2648                 t->metadata = metadata;
2649                 t->structs[p->metadata_struct_id] = metadata;
2650         }
2651
2652         return 0;
2653 }
2654
2655 static void
2656 metadata_build_free(struct rte_swx_pipeline *p)
2657 {
2658         uint32_t i;
2659
2660         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2661                 struct thread *t = &p->threads[i];
2662
2663                 free(t->metadata);
2664                 t->metadata = NULL;
2665         }
2666 }
2667
2668 static void
2669 metadata_free(struct rte_swx_pipeline *p)
2670 {
2671         metadata_build_free(p);
2672 }
2673
2674 /*
2675  * Instruction.
2676  */
2677 static int
2678 instruction_is_tx(enum instruction_type type)
2679 {
2680         switch (type) {
2681         case INSTR_TX:
2682         case INSTR_TX_I:
2683                 return 1;
2684
2685         default:
2686                 return 0;
2687         }
2688 }
2689
2690 static int
2691 instruction_is_jmp(struct instruction *instr)
2692 {
2693         switch (instr->type) {
2694         case INSTR_JMP:
2695         case INSTR_JMP_VALID:
2696         case INSTR_JMP_INVALID:
2697         case INSTR_JMP_HIT:
2698         case INSTR_JMP_MISS:
2699         case INSTR_JMP_ACTION_HIT:
2700         case INSTR_JMP_ACTION_MISS:
2701         case INSTR_JMP_EQ:
2702         case INSTR_JMP_EQ_MH:
2703         case INSTR_JMP_EQ_HM:
2704         case INSTR_JMP_EQ_HH:
2705         case INSTR_JMP_EQ_I:
2706         case INSTR_JMP_NEQ:
2707         case INSTR_JMP_NEQ_MH:
2708         case INSTR_JMP_NEQ_HM:
2709         case INSTR_JMP_NEQ_HH:
2710         case INSTR_JMP_NEQ_I:
2711         case INSTR_JMP_LT:
2712         case INSTR_JMP_LT_MH:
2713         case INSTR_JMP_LT_HM:
2714         case INSTR_JMP_LT_HH:
2715         case INSTR_JMP_LT_MI:
2716         case INSTR_JMP_LT_HI:
2717         case INSTR_JMP_GT:
2718         case INSTR_JMP_GT_MH:
2719         case INSTR_JMP_GT_HM:
2720         case INSTR_JMP_GT_HH:
2721         case INSTR_JMP_GT_MI:
2722         case INSTR_JMP_GT_HI:
2723                 return 1;
2724
2725         default:
2726                 return 0;
2727         }
2728 }
2729
2730 static struct field *
2731 action_field_parse(struct action *action, const char *name);
2732
2733 static struct field *
2734 struct_field_parse(struct rte_swx_pipeline *p,
2735                    struct action *action,
2736                    const char *name,
2737                    uint32_t *struct_id)
2738 {
2739         struct field *f;
2740
2741         switch (name[0]) {
2742         case 'h':
2743         {
2744                 struct header *header;
2745
2746                 f = header_field_parse(p, name, &header);
2747                 if (!f)
2748                         return NULL;
2749
2750                 *struct_id = header->struct_id;
2751                 return f;
2752         }
2753
2754         case 'm':
2755         {
2756                 f = metadata_field_parse(p, name);
2757                 if (!f)
2758                         return NULL;
2759
2760                 *struct_id = p->metadata_struct_id;
2761                 return f;
2762         }
2763
2764         case 't':
2765         {
2766                 if (!action)
2767                         return NULL;
2768
2769                 f = action_field_parse(action, name);
2770                 if (!f)
2771                         return NULL;
2772
2773                 *struct_id = 0;
2774                 return f;
2775         }
2776
2777         case 'e':
2778         {
2779                 struct extern_obj *obj;
2780
2781                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2782                 if (!f)
2783                         return NULL;
2784
2785                 *struct_id = obj->struct_id;
2786                 return f;
2787         }
2788
2789         case 'f':
2790         {
2791                 struct extern_func *func;
2792
2793                 f = extern_func_mailbox_field_parse(p, name, &func);
2794                 if (!f)
2795                         return NULL;
2796
2797                 *struct_id = func->struct_id;
2798                 return f;
2799         }
2800
2801         default:
2802                 return NULL;
2803         }
2804 }
2805
2806 static inline void
2807 pipeline_port_inc(struct rte_swx_pipeline *p)
2808 {
2809         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2810 }
2811
2812 static inline void
2813 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2814 {
2815         t->ip = p->instructions;
2816 }
2817
2818 static inline void
2819 thread_ip_set(struct thread *t, struct instruction *ip)
2820 {
2821         t->ip = ip;
2822 }
2823
2824 static inline void
2825 thread_ip_action_call(struct rte_swx_pipeline *p,
2826                       struct thread *t,
2827                       uint32_t action_id)
2828 {
2829         t->ret = t->ip + 1;
2830         t->ip = p->action_instructions[action_id];
2831 }
2832
2833 static inline void
2834 thread_ip_inc(struct rte_swx_pipeline *p);
2835
2836 static inline void
2837 thread_ip_inc(struct rte_swx_pipeline *p)
2838 {
2839         struct thread *t = &p->threads[p->thread_id];
2840
2841         t->ip++;
2842 }
2843
2844 static inline void
2845 thread_ip_inc_cond(struct thread *t, int cond)
2846 {
2847         t->ip += cond;
2848 }
2849
2850 static inline void
2851 thread_yield(struct rte_swx_pipeline *p)
2852 {
2853         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2854 }
2855
2856 static inline void
2857 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2858 {
2859         p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2860 }
2861
2862 /*
2863  * rx.
2864  */
2865 static int
2866 instr_rx_translate(struct rte_swx_pipeline *p,
2867                    struct action *action,
2868                    char **tokens,
2869                    int n_tokens,
2870                    struct instruction *instr,
2871                    struct instruction_data *data __rte_unused)
2872 {
2873         struct field *f;
2874
2875         CHECK(!action, EINVAL);
2876         CHECK(n_tokens == 2, EINVAL);
2877
2878         f = metadata_field_parse(p, tokens[1]);
2879         CHECK(f, EINVAL);
2880
2881         instr->type = INSTR_RX;
2882         instr->io.io.offset = f->offset / 8;
2883         instr->io.io.n_bits = f->n_bits;
2884         return 0;
2885 }
2886
2887 static inline void
2888 instr_rx_exec(struct rte_swx_pipeline *p);
2889
2890 static inline void
2891 instr_rx_exec(struct rte_swx_pipeline *p)
2892 {
2893         struct thread *t = &p->threads[p->thread_id];
2894         struct instruction *ip = t->ip;
2895         struct port_in_runtime *port = &p->in[p->port_id];
2896         struct rte_swx_pkt *pkt = &t->pkt;
2897         int pkt_received;
2898
2899         /* Packet. */
2900         pkt_received = port->pkt_rx(port->obj, pkt);
2901         t->ptr = &pkt->pkt[pkt->offset];
2902         rte_prefetch0(t->ptr);
2903
2904         TRACE("[Thread %2u] rx %s from port %u\n",
2905               p->thread_id,
2906               pkt_received ? "1 pkt" : "0 pkts",
2907               p->port_id);
2908
2909         /* Headers. */
2910         t->valid_headers = 0;
2911         t->n_headers_out = 0;
2912
2913         /* Meta-data. */
2914         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2915
2916         /* Tables. */
2917         t->table_state = p->table_state;
2918
2919         /* Thread. */
2920         pipeline_port_inc(p);
2921         thread_ip_inc_cond(t, pkt_received);
2922         thread_yield(p);
2923 }
2924
2925 /*
2926  * tx.
2927  */
2928 static int
2929 instr_tx_translate(struct rte_swx_pipeline *p,
2930                    struct action *action __rte_unused,
2931                    char **tokens,
2932                    int n_tokens,
2933                    struct instruction *instr,
2934                    struct instruction_data *data __rte_unused)
2935 {
2936         char *port = tokens[1];
2937         struct field *f;
2938         uint32_t port_val;
2939
2940         CHECK(n_tokens == 2, EINVAL);
2941
2942         f = metadata_field_parse(p, port);
2943         if (f) {
2944                 instr->type = INSTR_TX;
2945                 instr->io.io.offset = f->offset / 8;
2946                 instr->io.io.n_bits = f->n_bits;
2947                 return 0;
2948         }
2949
2950         /* TX_I. */
2951         port_val = strtoul(port, &port, 0);
2952         CHECK(!port[0], EINVAL);
2953
2954         instr->type = INSTR_TX_I;
2955         instr->io.io.val = port_val;
2956         return 0;
2957 }
2958
2959 static int
2960 instr_drop_translate(struct rte_swx_pipeline *p,
2961                      struct action *action __rte_unused,
2962                      char **tokens __rte_unused,
2963                      int n_tokens,
2964                      struct instruction *instr,
2965                      struct instruction_data *data __rte_unused)
2966 {
2967         CHECK(n_tokens == 1, EINVAL);
2968
2969         /* TX_I. */
2970         instr->type = INSTR_TX_I;
2971         instr->io.io.val = p->n_ports_out - 1;
2972         return 0;
2973 }
2974
2975 static inline void
2976 emit_handler(struct thread *t)
2977 {
2978         struct header_out_runtime *h0 = &t->headers_out[0];
2979         struct header_out_runtime *h1 = &t->headers_out[1];
2980         uint32_t offset = 0, i;
2981
2982         /* No header change or header decapsulation. */
2983         if ((t->n_headers_out == 1) &&
2984             (h0->ptr + h0->n_bytes == t->ptr)) {
2985                 TRACE("Emit handler: no header change or header decap.\n");
2986
2987                 t->pkt.offset -= h0->n_bytes;
2988                 t->pkt.length += h0->n_bytes;
2989
2990                 return;
2991         }
2992
2993         /* Header encapsulation (optionally, with prior header decasulation). */
2994         if ((t->n_headers_out == 2) &&
2995             (h1->ptr + h1->n_bytes == t->ptr) &&
2996             (h0->ptr == h0->ptr0)) {
2997                 uint32_t offset;
2998
2999                 TRACE("Emit handler: header encapsulation.\n");
3000
3001                 offset = h0->n_bytes + h1->n_bytes;
3002                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
3003                 t->pkt.offset -= offset;
3004                 t->pkt.length += offset;
3005
3006                 return;
3007         }
3008
3009         /* Header insertion. */
3010         /* TBD */
3011
3012         /* Header extraction. */
3013         /* TBD */
3014
3015         /* For any other case. */
3016         TRACE("Emit handler: complex case.\n");
3017
3018         for (i = 0; i < t->n_headers_out; i++) {
3019                 struct header_out_runtime *h = &t->headers_out[i];
3020
3021                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
3022                 offset += h->n_bytes;
3023         }
3024
3025         if (offset) {
3026                 memcpy(t->ptr - offset, t->header_out_storage, offset);
3027                 t->pkt.offset -= offset;
3028                 t->pkt.length += offset;
3029         }
3030 }
3031
3032 static inline void
3033 instr_tx_exec(struct rte_swx_pipeline *p);
3034
3035 static inline void
3036 instr_tx_exec(struct rte_swx_pipeline *p)
3037 {
3038         struct thread *t = &p->threads[p->thread_id];
3039         struct instruction *ip = t->ip;
3040         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
3041         struct port_out_runtime *port = &p->out[port_id];
3042         struct rte_swx_pkt *pkt = &t->pkt;
3043
3044         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
3045               p->thread_id,
3046               (uint32_t)port_id);
3047
3048         /* Headers. */
3049         emit_handler(t);
3050
3051         /* Packet. */
3052         port->pkt_tx(port->obj, pkt);
3053
3054         /* Thread. */
3055         thread_ip_reset(p, t);
3056         instr_rx_exec(p);
3057 }
3058
3059 static inline void
3060 instr_tx_i_exec(struct rte_swx_pipeline *p)
3061 {
3062         struct thread *t = &p->threads[p->thread_id];
3063         struct instruction *ip = t->ip;
3064         uint64_t port_id = ip->io.io.val;
3065         struct port_out_runtime *port = &p->out[port_id];
3066         struct rte_swx_pkt *pkt = &t->pkt;
3067
3068         TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
3069               p->thread_id,
3070               (uint32_t)port_id);
3071
3072         /* Headers. */
3073         emit_handler(t);
3074
3075         /* Packet. */
3076         port->pkt_tx(port->obj, pkt);
3077
3078         /* Thread. */
3079         thread_ip_reset(p, t);
3080         instr_rx_exec(p);
3081 }
3082
3083 /*
3084  * extract.
3085  */
3086 static int
3087 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
3088                             struct action *action,
3089                             char **tokens,
3090                             int n_tokens,
3091                             struct instruction *instr,
3092                             struct instruction_data *data __rte_unused)
3093 {
3094         struct header *h;
3095
3096         CHECK(!action, EINVAL);
3097         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
3098
3099         h = header_parse(p, tokens[1]);
3100         CHECK(h, EINVAL);
3101
3102         if (n_tokens == 2) {
3103                 CHECK(!h->st->var_size, EINVAL);
3104
3105                 instr->type = INSTR_HDR_EXTRACT;
3106                 instr->io.hdr.header_id[0] = h->id;
3107                 instr->io.hdr.struct_id[0] = h->struct_id;
3108                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3109         } else {
3110                 struct field *mf;
3111
3112                 CHECK(h->st->var_size, EINVAL);
3113
3114                 mf = metadata_field_parse(p, tokens[2]);
3115                 CHECK(mf, EINVAL);
3116                 CHECK(!mf->var_size, EINVAL);
3117
3118                 instr->type = INSTR_HDR_EXTRACT_M;
3119                 instr->io.io.offset = mf->offset / 8;
3120                 instr->io.io.n_bits = mf->n_bits;
3121                 instr->io.hdr.header_id[0] = h->id;
3122                 instr->io.hdr.struct_id[0] = h->struct_id;
3123                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
3124         }
3125
3126         return 0;
3127 }
3128
3129 static int
3130 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
3131                               struct action *action,
3132                               char **tokens,
3133                               int n_tokens,
3134                               struct instruction *instr,
3135                               struct instruction_data *data __rte_unused)
3136 {
3137         struct header *h;
3138
3139         CHECK(!action, EINVAL);
3140         CHECK(n_tokens == 2, EINVAL);
3141
3142         h = header_parse(p, tokens[1]);
3143         CHECK(h, EINVAL);
3144         CHECK(!h->st->var_size, EINVAL);
3145
3146         instr->type = INSTR_HDR_LOOKAHEAD;
3147         instr->io.hdr.header_id[0] = h->id;
3148         instr->io.hdr.struct_id[0] = h->struct_id;
3149         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
3150
3151         return 0;
3152 }
3153
3154 static inline void
3155 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
3156
3157 static inline void
3158 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
3159 {
3160         struct thread *t = &p->threads[p->thread_id];
3161         struct instruction *ip = t->ip;
3162         uint64_t valid_headers = t->valid_headers;
3163         uint8_t *ptr = t->ptr;
3164         uint32_t offset = t->pkt.offset;
3165         uint32_t length = t->pkt.length;
3166         uint32_t i;
3167
3168         for (i = 0; i < n_extract; i++) {
3169                 uint32_t header_id = ip->io.hdr.header_id[i];
3170                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3171                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3172
3173                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
3174                       p->thread_id,
3175                       header_id,
3176                       n_bytes);
3177
3178                 /* Headers. */
3179                 t->structs[struct_id] = ptr;
3180                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3181
3182                 /* Packet. */
3183                 offset += n_bytes;
3184                 length -= n_bytes;
3185                 ptr += n_bytes;
3186         }
3187
3188         /* Headers. */
3189         t->valid_headers = valid_headers;
3190
3191         /* Packet. */
3192         t->pkt.offset = offset;
3193         t->pkt.length = length;
3194         t->ptr = ptr;
3195 }
3196
3197 static inline void
3198 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
3199 {
3200         __instr_hdr_extract_exec(p, 1);
3201
3202         /* Thread. */
3203         thread_ip_inc(p);
3204 }
3205
3206 static inline void
3207 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
3208 {
3209         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3210               p->thread_id);
3211
3212         __instr_hdr_extract_exec(p, 2);
3213
3214         /* Thread. */
3215         thread_ip_inc(p);
3216 }
3217
3218 static inline void
3219 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
3220 {
3221         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3222               p->thread_id);
3223
3224         __instr_hdr_extract_exec(p, 3);
3225
3226         /* Thread. */
3227         thread_ip_inc(p);
3228 }
3229
3230 static inline void
3231 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
3232 {
3233         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3234               p->thread_id);
3235
3236         __instr_hdr_extract_exec(p, 4);
3237
3238         /* Thread. */
3239         thread_ip_inc(p);
3240 }
3241
3242 static inline void
3243 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
3244 {
3245         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3246               p->thread_id);
3247
3248         __instr_hdr_extract_exec(p, 5);
3249
3250         /* Thread. */
3251         thread_ip_inc(p);
3252 }
3253
3254 static inline void
3255 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
3256 {
3257         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3258               p->thread_id);
3259
3260         __instr_hdr_extract_exec(p, 6);
3261
3262         /* Thread. */
3263         thread_ip_inc(p);
3264 }
3265
3266 static inline void
3267 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
3268 {
3269         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3270               p->thread_id);
3271
3272         __instr_hdr_extract_exec(p, 7);
3273
3274         /* Thread. */
3275         thread_ip_inc(p);
3276 }
3277
3278 static inline void
3279 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
3280 {
3281         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3282               p->thread_id);
3283
3284         __instr_hdr_extract_exec(p, 8);
3285
3286         /* Thread. */
3287         thread_ip_inc(p);
3288 }
3289
3290 static inline void
3291 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
3292 {
3293         struct thread *t = &p->threads[p->thread_id];
3294         struct instruction *ip = t->ip;
3295
3296         uint64_t valid_headers = t->valid_headers;
3297         uint8_t *ptr = t->ptr;
3298         uint32_t offset = t->pkt.offset;
3299         uint32_t length = t->pkt.length;
3300
3301         uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
3302         uint32_t header_id = ip->io.hdr.header_id[0];
3303         uint32_t struct_id = ip->io.hdr.struct_id[0];
3304         uint32_t n_bytes = ip->io.hdr.n_bytes[0];
3305
3306         struct header_runtime *h = &t->headers[header_id];
3307
3308         TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
3309               p->thread_id,
3310               header_id,
3311               n_bytes,
3312               n_bytes_last);
3313
3314         n_bytes += n_bytes_last;
3315
3316         /* Headers. */
3317         t->structs[struct_id] = ptr;
3318         t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3319         h->n_bytes = n_bytes;
3320
3321         /* Packet. */
3322         t->pkt.offset = offset + n_bytes;
3323         t->pkt.length = length - n_bytes;
3324         t->ptr = ptr + n_bytes;
3325
3326         /* Thread. */
3327         thread_ip_inc(p);
3328 }
3329
3330 static inline void
3331 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
3332 {
3333         struct thread *t = &p->threads[p->thread_id];
3334         struct instruction *ip = t->ip;
3335
3336         uint64_t valid_headers = t->valid_headers;
3337         uint8_t *ptr = t->ptr;
3338
3339         uint32_t header_id = ip->io.hdr.header_id[0];
3340         uint32_t struct_id = ip->io.hdr.struct_id[0];
3341
3342         TRACE("[Thread %2u]: lookahead header %u\n",
3343               p->thread_id,
3344               header_id);
3345
3346         /* Headers. */
3347         t->structs[struct_id] = ptr;
3348         t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3349
3350         /* Thread. */
3351         thread_ip_inc(p);
3352 }
3353
3354 /*
3355  * emit.
3356  */
3357 static int
3358 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
3359                          struct action *action __rte_unused,
3360                          char **tokens,
3361                          int n_tokens,
3362                          struct instruction *instr,
3363                          struct instruction_data *data __rte_unused)
3364 {
3365         struct header *h;
3366
3367         CHECK(n_tokens == 2, EINVAL);
3368
3369         h = header_parse(p, tokens[1]);
3370         CHECK(h, EINVAL);
3371
3372         instr->type = INSTR_HDR_EMIT;
3373         instr->io.hdr.header_id[0] = h->id;
3374         instr->io.hdr.struct_id[0] = h->struct_id;
3375         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3376         return 0;
3377 }
3378
3379 static inline void
3380 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
3381
3382 static inline void
3383 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
3384 {
3385         struct thread *t = &p->threads[p->thread_id];
3386         struct instruction *ip = t->ip;
3387         uint64_t valid_headers = t->valid_headers;
3388         uint32_t n_headers_out = t->n_headers_out;
3389         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
3390         uint8_t *ho_ptr = NULL;
3391         uint32_t ho_nbytes = 0, first = 1, i;
3392
3393         for (i = 0; i < n_emit; i++) {
3394                 uint32_t header_id = ip->io.hdr.header_id[i];
3395                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3396
3397                 struct header_runtime *hi = &t->headers[header_id];
3398                 uint8_t *hi_ptr0 = hi->ptr0;
3399                 uint32_t n_bytes = hi->n_bytes;
3400
3401                 uint8_t *hi_ptr = t->structs[struct_id];
3402
3403                 if (!MASK64_BIT_GET(valid_headers, header_id))
3404                         continue;
3405
3406                 TRACE("[Thread %2u]: emit header %u\n",
3407                       p->thread_id,
3408                       header_id);
3409
3410                 /* Headers. */
3411                 if (first) {
3412                         first = 0;
3413
3414                         if (!t->n_headers_out) {
3415                                 ho = &t->headers_out[0];
3416
3417                                 ho->ptr0 = hi_ptr0;
3418                                 ho->ptr = hi_ptr;
3419
3420                                 ho_ptr = hi_ptr;
3421                                 ho_nbytes = n_bytes;
3422
3423                                 n_headers_out = 1;
3424
3425                                 continue;
3426                         } else {
3427                                 ho_ptr = ho->ptr;
3428                                 ho_nbytes = ho->n_bytes;
3429                         }
3430                 }
3431
3432                 if (ho_ptr + ho_nbytes == hi_ptr) {
3433                         ho_nbytes += n_bytes;
3434                 } else {
3435                         ho->n_bytes = ho_nbytes;
3436
3437                         ho++;
3438                         ho->ptr0 = hi_ptr0;
3439                         ho->ptr = hi_ptr;
3440
3441                         ho_ptr = hi_ptr;
3442                         ho_nbytes = n_bytes;
3443
3444                         n_headers_out++;
3445                 }
3446         }
3447
3448         ho->n_bytes = ho_nbytes;
3449         t->n_headers_out = n_headers_out;
3450 }
3451
3452 static inline void
3453 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
3454 {
3455         __instr_hdr_emit_exec(p, 1);
3456
3457         /* Thread. */
3458         thread_ip_inc(p);
3459 }
3460
3461 static inline void
3462 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
3463 {
3464         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3465               p->thread_id);
3466
3467         __instr_hdr_emit_exec(p, 1);
3468         instr_tx_exec(p);
3469 }
3470
3471 static inline void
3472 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
3473 {
3474         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3475               p->thread_id);
3476
3477         __instr_hdr_emit_exec(p, 2);
3478         instr_tx_exec(p);
3479 }
3480
3481 static inline void
3482 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
3483 {
3484         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3485               p->thread_id);
3486
3487         __instr_hdr_emit_exec(p, 3);
3488         instr_tx_exec(p);
3489 }
3490
3491 static inline void
3492 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
3493 {
3494         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3495               p->thread_id);
3496
3497         __instr_hdr_emit_exec(p, 4);
3498         instr_tx_exec(p);
3499 }
3500
3501 static inline void
3502 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
3503 {
3504         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3505               p->thread_id);
3506
3507         __instr_hdr_emit_exec(p, 5);
3508         instr_tx_exec(p);
3509 }
3510
3511 static inline void
3512 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
3513 {
3514         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3515               p->thread_id);
3516
3517         __instr_hdr_emit_exec(p, 6);
3518         instr_tx_exec(p);
3519 }
3520
3521 static inline void
3522 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
3523 {
3524         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3525               p->thread_id);
3526
3527         __instr_hdr_emit_exec(p, 7);
3528         instr_tx_exec(p);
3529 }
3530
3531 static inline void
3532 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
3533 {
3534         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
3535               p->thread_id);
3536
3537         __instr_hdr_emit_exec(p, 8);
3538         instr_tx_exec(p);
3539 }
3540
3541 /*
3542  * validate.
3543  */
3544 static int
3545 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
3546                              struct action *action __rte_unused,
3547                              char **tokens,
3548                              int n_tokens,
3549                              struct instruction *instr,
3550                              struct instruction_data *data __rte_unused)
3551 {
3552         struct header *h;
3553
3554         CHECK(n_tokens == 2, EINVAL);
3555
3556         h = header_parse(p, tokens[1]);
3557         CHECK(h, EINVAL);
3558
3559         instr->type = INSTR_HDR_VALIDATE;
3560         instr->valid.header_id = h->id;
3561         return 0;
3562 }
3563
3564 static inline void
3565 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
3566 {
3567         struct thread *t = &p->threads[p->thread_id];
3568         struct instruction *ip = t->ip;
3569         uint32_t header_id = ip->valid.header_id;
3570
3571         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
3572
3573         /* Headers. */
3574         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
3575
3576         /* Thread. */
3577         thread_ip_inc(p);
3578 }
3579
3580 /*
3581  * invalidate.
3582  */
3583 static int
3584 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
3585                                struct action *action __rte_unused,
3586                                char **tokens,
3587                                int n_tokens,
3588                                struct instruction *instr,
3589                                struct instruction_data *data __rte_unused)
3590 {
3591         struct header *h;
3592
3593         CHECK(n_tokens == 2, EINVAL);
3594
3595         h = header_parse(p, tokens[1]);
3596         CHECK(h, EINVAL);
3597
3598         instr->type = INSTR_HDR_INVALIDATE;
3599         instr->valid.header_id = h->id;
3600         return 0;
3601 }
3602
3603 static inline void
3604 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3605 {
3606         struct thread *t = &p->threads[p->thread_id];
3607         struct instruction *ip = t->ip;
3608         uint32_t header_id = ip->valid.header_id;
3609
3610         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3611
3612         /* Headers. */
3613         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3614
3615         /* Thread. */
3616         thread_ip_inc(p);
3617 }
3618
3619 /*
3620  * table.
3621  */
3622 static struct table *
3623 table_find(struct rte_swx_pipeline *p, const char *name);
3624
3625 static struct selector *
3626 selector_find(struct rte_swx_pipeline *p, const char *name);
3627
3628 static int
3629 instr_table_translate(struct rte_swx_pipeline *p,
3630                       struct action *action,
3631                       char **tokens,
3632                       int n_tokens,
3633                       struct instruction *instr,
3634                       struct instruction_data *data __rte_unused)
3635 {
3636         struct table *t;
3637         struct selector *s;
3638
3639         CHECK(!action, EINVAL);
3640         CHECK(n_tokens == 2, EINVAL);
3641
3642         t = table_find(p, tokens[1]);
3643         if (t) {
3644                 instr->type = INSTR_TABLE;
3645                 instr->table.table_id = t->id;
3646                 return 0;
3647         }
3648
3649         s = selector_find(p, tokens[1]);
3650         if (s) {
3651                 instr->type = INSTR_SELECTOR;
3652                 instr->table.table_id = s->id;
3653                 return 0;
3654         }
3655
3656         CHECK(0, EINVAL);
3657 }
3658
3659 static inline void
3660 instr_table_exec(struct rte_swx_pipeline *p)
3661 {
3662         struct thread *t = &p->threads[p->thread_id];
3663         struct instruction *ip = t->ip;
3664         uint32_t table_id = ip->table.table_id;
3665         struct rte_swx_table_state *ts = &t->table_state[table_id];
3666         struct table_runtime *table = &t->tables[table_id];
3667         struct table_statistics *stats = &p->table_stats[table_id];
3668         uint64_t action_id, n_pkts_hit, n_pkts_action;
3669         uint8_t *action_data;
3670         int done, hit;
3671
3672         /* Table. */
3673         done = table->func(ts->obj,
3674                            table->mailbox,
3675                            table->key,
3676                            &action_id,
3677                            &action_data,
3678                            &hit);
3679         if (!done) {
3680                 /* Thread. */
3681                 TRACE("[Thread %2u] table %u (not finalized)\n",
3682                       p->thread_id,
3683                       table_id);
3684
3685                 thread_yield(p);
3686                 return;
3687         }
3688
3689         action_id = hit ? action_id : ts->default_action_id;
3690         action_data = hit ? action_data : ts->default_action_data;
3691         n_pkts_hit = stats->n_pkts_hit[hit];
3692         n_pkts_action = stats->n_pkts_action[action_id];
3693
3694         TRACE("[Thread %2u] table %u (%s, action %u)\n",
3695               p->thread_id,
3696               table_id,
3697               hit ? "hit" : "miss",
3698               (uint32_t)action_id);
3699
3700         t->action_id = action_id;
3701         t->structs[0] = action_data;
3702         t->hit = hit;
3703         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
3704         stats->n_pkts_action[action_id] = n_pkts_action + 1;
3705
3706         /* Thread. */
3707         thread_ip_action_call(p, t, action_id);
3708 }
3709
3710 static inline void
3711 instr_selector_exec(struct rte_swx_pipeline *p)
3712 {
3713         struct thread *t = &p->threads[p->thread_id];
3714         struct instruction *ip = t->ip;
3715         uint32_t selector_id = ip->table.table_id;
3716         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
3717         struct selector_runtime *selector = &t->selectors[selector_id];
3718         struct selector_statistics *stats = &p->selector_stats[selector_id];
3719         uint64_t n_pkts = stats->n_pkts;
3720         int done;
3721
3722         /* Table. */
3723         done = rte_swx_table_selector_select(ts->obj,
3724                            selector->mailbox,
3725                            selector->group_id_buffer,
3726                            selector->selector_buffer,
3727                            selector->member_id_buffer);
3728         if (!done) {
3729                 /* Thread. */
3730                 TRACE("[Thread %2u] selector %u (not finalized)\n",
3731                       p->thread_id,
3732                       selector_id);
3733
3734                 thread_yield(p);
3735                 return;
3736         }
3737
3738
3739         TRACE("[Thread %2u] selector %u\n",
3740               p->thread_id,
3741               selector_id);
3742
3743         stats->n_pkts = n_pkts + 1;
3744
3745         /* Thread. */
3746         thread_ip_inc(p);
3747 }
3748
3749 /*
3750  * extern.
3751  */
3752 static int
3753 instr_extern_translate(struct rte_swx_pipeline *p,
3754                        struct action *action __rte_unused,
3755                        char **tokens,
3756                        int n_tokens,
3757                        struct instruction *instr,
3758                        struct instruction_data *data __rte_unused)
3759 {
3760         char *token = tokens[1];
3761
3762         CHECK(n_tokens == 2, EINVAL);
3763
3764         if (token[0] == 'e') {
3765                 struct extern_obj *obj;
3766                 struct extern_type_member_func *func;
3767
3768                 func = extern_obj_member_func_parse(p, token, &obj);
3769                 CHECK(func, EINVAL);
3770
3771                 instr->type = INSTR_EXTERN_OBJ;
3772                 instr->ext_obj.ext_obj_id = obj->id;
3773                 instr->ext_obj.func_id = func->id;
3774
3775                 return 0;
3776         }
3777
3778         if (token[0] == 'f') {
3779                 struct extern_func *func;
3780
3781                 func = extern_func_parse(p, token);
3782                 CHECK(func, EINVAL);
3783
3784                 instr->type = INSTR_EXTERN_FUNC;
3785                 instr->ext_func.ext_func_id = func->id;
3786
3787                 return 0;
3788         }
3789
3790         CHECK(0, EINVAL);
3791 }
3792
3793 static inline void
3794 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3795 {
3796         struct thread *t = &p->threads[p->thread_id];
3797         struct instruction *ip = t->ip;
3798         uint32_t obj_id = ip->ext_obj.ext_obj_id;
3799         uint32_t func_id = ip->ext_obj.func_id;
3800         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3801         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3802
3803         TRACE("[Thread %2u] extern obj %u member func %u\n",
3804               p->thread_id,
3805               obj_id,
3806               func_id);
3807
3808         /* Extern object member function execute. */
3809         uint32_t done = func(obj->obj, obj->mailbox);
3810
3811         /* Thread. */
3812         thread_ip_inc_cond(t, done);
3813         thread_yield_cond(p, done ^ 1);
3814 }
3815
3816 static inline void
3817 instr_extern_func_exec(struct rte_swx_pipeline *p)
3818 {
3819         struct thread *t = &p->threads[p->thread_id];
3820         struct instruction *ip = t->ip;
3821         uint32_t ext_func_id = ip->ext_func.ext_func_id;
3822         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3823         rte_swx_extern_func_t func = ext_func->func;
3824
3825         TRACE("[Thread %2u] extern func %u\n",
3826               p->thread_id,
3827               ext_func_id);
3828
3829         /* Extern function execute. */
3830         uint32_t done = func(ext_func->mailbox);
3831
3832         /* Thread. */
3833         thread_ip_inc_cond(t, done);
3834         thread_yield_cond(p, done ^ 1);
3835 }
3836
3837 /*
3838  * mov.
3839  */
3840 static int
3841 instr_mov_translate(struct rte_swx_pipeline *p,
3842                     struct action *action,
3843                     char **tokens,
3844                     int n_tokens,
3845                     struct instruction *instr,
3846                     struct instruction_data *data __rte_unused)
3847 {
3848         char *dst = tokens[1], *src = tokens[2];
3849         struct field *fdst, *fsrc;
3850         uint64_t src_val;
3851         uint32_t dst_struct_id = 0, src_struct_id = 0;
3852
3853         CHECK(n_tokens == 3, EINVAL);
3854
3855         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3856         CHECK(fdst, EINVAL);
3857         CHECK(!fdst->var_size, EINVAL);
3858
3859         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
3860         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3861         if (fsrc) {
3862                 CHECK(!fsrc->var_size, EINVAL);
3863
3864                 instr->type = INSTR_MOV;
3865                 if (dst[0] != 'h' && src[0] == 'h')
3866                         instr->type = INSTR_MOV_MH;
3867                 if (dst[0] == 'h' && src[0] != 'h')
3868                         instr->type = INSTR_MOV_HM;
3869                 if (dst[0] == 'h' && src[0] == 'h')
3870                         instr->type = INSTR_MOV_HH;
3871
3872                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3873                 instr->mov.dst.n_bits = fdst->n_bits;
3874                 instr->mov.dst.offset = fdst->offset / 8;
3875                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3876                 instr->mov.src.n_bits = fsrc->n_bits;
3877                 instr->mov.src.offset = fsrc->offset / 8;
3878                 return 0;
3879         }
3880
3881         /* MOV_I. */
3882         src_val = strtoull(src, &src, 0);
3883         CHECK(!src[0], EINVAL);
3884
3885         if (dst[0] == 'h')
3886                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3887
3888         instr->type = INSTR_MOV_I;
3889         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3890         instr->mov.dst.n_bits = fdst->n_bits;
3891         instr->mov.dst.offset = fdst->offset / 8;
3892         instr->mov.src_val = src_val;
3893         return 0;
3894 }
3895
3896 static inline void
3897 instr_mov_exec(struct rte_swx_pipeline *p)
3898 {
3899         struct thread *t = &p->threads[p->thread_id];
3900         struct instruction *ip = t->ip;
3901
3902         TRACE("[Thread %2u] mov\n",
3903               p->thread_id);
3904
3905         MOV(t, ip);
3906
3907         /* Thread. */
3908         thread_ip_inc(p);
3909 }
3910
3911 static inline void
3912 instr_mov_mh_exec(struct rte_swx_pipeline *p)
3913 {
3914         struct thread *t = &p->threads[p->thread_id];
3915         struct instruction *ip = t->ip;
3916
3917         TRACE("[Thread %2u] mov (mh)\n",
3918               p->thread_id);
3919
3920         MOV_MH(t, ip);
3921
3922         /* Thread. */
3923         thread_ip_inc(p);
3924 }
3925
3926 static inline void
3927 instr_mov_hm_exec(struct rte_swx_pipeline *p)
3928 {
3929         struct thread *t = &p->threads[p->thread_id];
3930         struct instruction *ip = t->ip;
3931
3932         TRACE("[Thread %2u] mov (hm)\n",
3933               p->thread_id);
3934
3935         MOV_HM(t, ip);
3936
3937         /* Thread. */
3938         thread_ip_inc(p);
3939 }
3940
3941 static inline void
3942 instr_mov_hh_exec(struct rte_swx_pipeline *p)
3943 {
3944         struct thread *t = &p->threads[p->thread_id];
3945         struct instruction *ip = t->ip;
3946
3947         TRACE("[Thread %2u] mov (hh)\n",
3948               p->thread_id);
3949
3950         MOV_HH(t, ip);
3951
3952         /* Thread. */
3953         thread_ip_inc(p);
3954 }
3955
3956 static inline void
3957 instr_mov_i_exec(struct rte_swx_pipeline *p)
3958 {
3959         struct thread *t = &p->threads[p->thread_id];
3960         struct instruction *ip = t->ip;
3961
3962         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
3963               p->thread_id,
3964               ip->mov.src_val);
3965
3966         MOV_I(t, ip);
3967
3968         /* Thread. */
3969         thread_ip_inc(p);
3970 }
3971
3972 /*
3973  * dma.
3974  */
3975 static inline void
3976 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3977
3978 static inline void
3979 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3980 {
3981         struct thread *t = &p->threads[p->thread_id];
3982         struct instruction *ip = t->ip;
3983         uint8_t *action_data = t->structs[0];
3984         uint64_t valid_headers = t->valid_headers;
3985         uint32_t i;
3986
3987         for (i = 0; i < n_dma; i++) {
3988                 uint32_t header_id = ip->dma.dst.header_id[i];
3989                 uint32_t struct_id = ip->dma.dst.struct_id[i];
3990                 uint32_t offset = ip->dma.src.offset[i];
3991                 uint32_t n_bytes = ip->dma.n_bytes[i];
3992
3993                 struct header_runtime *h = &t->headers[header_id];
3994                 uint8_t *h_ptr0 = h->ptr0;
3995                 uint8_t *h_ptr = t->structs[struct_id];
3996
3997                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3998                         h_ptr : h_ptr0;
3999                 void *src = &action_data[offset];
4000
4001                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
4002
4003                 /* Headers. */
4004                 memcpy(dst, src, n_bytes);
4005                 t->structs[struct_id] = dst;
4006                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
4007         }
4008
4009         t->valid_headers = valid_headers;
4010 }
4011
4012 static inline void
4013 instr_dma_ht_exec(struct rte_swx_pipeline *p)
4014 {
4015         __instr_dma_ht_exec(p, 1);
4016
4017         /* Thread. */
4018         thread_ip_inc(p);
4019 }
4020
4021 static inline void
4022 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
4023 {
4024         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
4025               p->thread_id);
4026
4027         __instr_dma_ht_exec(p, 2);
4028
4029         /* Thread. */
4030         thread_ip_inc(p);
4031 }
4032
4033 static inline void
4034 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
4035 {
4036         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
4037               p->thread_id);
4038
4039         __instr_dma_ht_exec(p, 3);
4040
4041         /* Thread. */
4042         thread_ip_inc(p);
4043 }
4044
4045 static inline void
4046 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
4047 {
4048         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
4049               p->thread_id);
4050
4051         __instr_dma_ht_exec(p, 4);
4052
4053         /* Thread. */
4054         thread_ip_inc(p);
4055 }
4056
4057 static inline void
4058 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
4059 {
4060         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
4061               p->thread_id);
4062
4063         __instr_dma_ht_exec(p, 5);
4064
4065         /* Thread. */
4066         thread_ip_inc(p);
4067 }
4068
4069 static inline void
4070 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
4071 {
4072         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
4073               p->thread_id);
4074
4075         __instr_dma_ht_exec(p, 6);
4076
4077         /* Thread. */
4078         thread_ip_inc(p);
4079 }
4080
4081 static inline void
4082 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
4083 {
4084         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
4085               p->thread_id);
4086
4087         __instr_dma_ht_exec(p, 7);
4088
4089         /* Thread. */
4090         thread_ip_inc(p);
4091 }
4092
4093 static inline void
4094 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
4095 {
4096         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
4097               p->thread_id);
4098
4099         __instr_dma_ht_exec(p, 8);
4100
4101         /* Thread. */
4102         thread_ip_inc(p);
4103 }
4104
4105 /*
4106  * alu.
4107  */
4108 static int
4109 instr_alu_add_translate(struct rte_swx_pipeline *p,
4110                         struct action *action,
4111                         char **tokens,
4112                         int n_tokens,
4113                         struct instruction *instr,
4114                         struct instruction_data *data __rte_unused)
4115 {
4116         char *dst = tokens[1], *src = tokens[2];
4117         struct field *fdst, *fsrc;
4118         uint64_t src_val;
4119         uint32_t dst_struct_id = 0, src_struct_id = 0;
4120
4121         CHECK(n_tokens == 3, EINVAL);
4122
4123         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4124         CHECK(fdst, EINVAL);
4125         CHECK(!fdst->var_size, EINVAL);
4126
4127         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
4128         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4129         if (fsrc) {
4130                 CHECK(!fsrc->var_size, EINVAL);
4131
4132                 instr->type = INSTR_ALU_ADD;
4133                 if (dst[0] == 'h' && src[0] != 'h')
4134                         instr->type = INSTR_ALU_ADD_HM;
4135                 if (dst[0] != 'h' && src[0] == 'h')
4136                         instr->type = INSTR_ALU_ADD_MH;
4137                 if (dst[0] == 'h' && src[0] == 'h')
4138                         instr->type = INSTR_ALU_ADD_HH;
4139
4140                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4141                 instr->alu.dst.n_bits = fdst->n_bits;
4142                 instr->alu.dst.offset = fdst->offset / 8;
4143                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4144                 instr->alu.src.n_bits = fsrc->n_bits;
4145                 instr->alu.src.offset = fsrc->offset / 8;
4146                 return 0;
4147         }
4148
4149         /* ADD_MI, ADD_HI. */
4150         src_val = strtoull(src, &src, 0);
4151         CHECK(!src[0], EINVAL);
4152
4153         instr->type = INSTR_ALU_ADD_MI;
4154         if (dst[0] == 'h')
4155                 instr->type = INSTR_ALU_ADD_HI;
4156
4157         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4158         instr->alu.dst.n_bits = fdst->n_bits;
4159         instr->alu.dst.offset = fdst->offset / 8;
4160         instr->alu.src_val = src_val;
4161         return 0;
4162 }
4163
4164 static int
4165 instr_alu_sub_translate(struct rte_swx_pipeline *p,
4166                         struct action *action,
4167                         char **tokens,
4168                         int n_tokens,
4169                         struct instruction *instr,
4170                         struct instruction_data *data __rte_unused)
4171 {
4172         char *dst = tokens[1], *src = tokens[2];
4173         struct field *fdst, *fsrc;
4174         uint64_t src_val;
4175         uint32_t dst_struct_id = 0, src_struct_id = 0;
4176
4177         CHECK(n_tokens == 3, EINVAL);
4178
4179         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4180         CHECK(fdst, EINVAL);
4181         CHECK(!fdst->var_size, EINVAL);
4182
4183         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
4184         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4185         if (fsrc) {
4186                 CHECK(!fsrc->var_size, EINVAL);
4187
4188                 instr->type = INSTR_ALU_SUB;
4189                 if (dst[0] == 'h' && src[0] != 'h')
4190                         instr->type = INSTR_ALU_SUB_HM;
4191                 if (dst[0] != 'h' && src[0] == 'h')
4192                         instr->type = INSTR_ALU_SUB_MH;
4193                 if (dst[0] == 'h' && src[0] == 'h')
4194                         instr->type = INSTR_ALU_SUB_HH;
4195
4196                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4197                 instr->alu.dst.n_bits = fdst->n_bits;
4198                 instr->alu.dst.offset = fdst->offset / 8;
4199                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4200                 instr->alu.src.n_bits = fsrc->n_bits;
4201                 instr->alu.src.offset = fsrc->offset / 8;
4202                 return 0;
4203         }
4204
4205         /* SUB_MI, SUB_HI. */
4206         src_val = strtoull(src, &src, 0);
4207         CHECK(!src[0], EINVAL);
4208
4209         instr->type = INSTR_ALU_SUB_MI;
4210         if (dst[0] == 'h')
4211                 instr->type = INSTR_ALU_SUB_HI;
4212
4213         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4214         instr->alu.dst.n_bits = fdst->n_bits;
4215         instr->alu.dst.offset = fdst->offset / 8;
4216         instr->alu.src_val = src_val;
4217         return 0;
4218 }
4219
4220 static int
4221 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
4222                           struct action *action __rte_unused,
4223                           char **tokens,
4224                           int n_tokens,
4225                           struct instruction *instr,
4226                           struct instruction_data *data __rte_unused)
4227 {
4228         char *dst = tokens[1], *src = tokens[2];
4229         struct header *hdst, *hsrc;
4230         struct field *fdst, *fsrc;
4231
4232         CHECK(n_tokens == 3, EINVAL);
4233
4234         fdst = header_field_parse(p, dst, &hdst);
4235         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4236         CHECK(!fdst->var_size, EINVAL);
4237
4238         /* CKADD_FIELD. */
4239         fsrc = header_field_parse(p, src, &hsrc);
4240         if (fsrc) {
4241                 CHECK(!fsrc->var_size, EINVAL);
4242
4243                 instr->type = INSTR_ALU_CKADD_FIELD;
4244                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4245                 instr->alu.dst.n_bits = fdst->n_bits;
4246                 instr->alu.dst.offset = fdst->offset / 8;
4247                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4248                 instr->alu.src.n_bits = fsrc->n_bits;
4249                 instr->alu.src.offset = fsrc->offset / 8;
4250                 return 0;
4251         }
4252
4253         /* CKADD_STRUCT, CKADD_STRUCT20. */
4254         hsrc = header_parse(p, src);
4255         CHECK(hsrc, EINVAL);
4256         CHECK(!hsrc->st->var_size, EINVAL);
4257
4258         instr->type = INSTR_ALU_CKADD_STRUCT;
4259         if ((hsrc->st->n_bits / 8) == 20)
4260                 instr->type = INSTR_ALU_CKADD_STRUCT20;
4261
4262         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4263         instr->alu.dst.n_bits = fdst->n_bits;
4264         instr->alu.dst.offset = fdst->offset / 8;
4265         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4266         instr->alu.src.n_bits = hsrc->st->n_bits;
4267         instr->alu.src.offset = 0; /* Unused. */
4268         return 0;
4269 }
4270
4271 static int
4272 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
4273                           struct action *action __rte_unused,
4274                           char **tokens,
4275                           int n_tokens,
4276                           struct instruction *instr,
4277                           struct instruction_data *data __rte_unused)
4278 {
4279         char *dst = tokens[1], *src = tokens[2];
4280         struct header *hdst, *hsrc;
4281         struct field *fdst, *fsrc;
4282
4283         CHECK(n_tokens == 3, EINVAL);
4284
4285         fdst = header_field_parse(p, dst, &hdst);
4286         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4287         CHECK(!fdst->var_size, EINVAL);
4288
4289         fsrc = header_field_parse(p, src, &hsrc);
4290         CHECK(fsrc, EINVAL);
4291         CHECK(!fsrc->var_size, EINVAL);
4292
4293         instr->type = INSTR_ALU_CKSUB_FIELD;
4294         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4295         instr->alu.dst.n_bits = fdst->n_bits;
4296         instr->alu.dst.offset = fdst->offset / 8;
4297         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4298         instr->alu.src.n_bits = fsrc->n_bits;
4299         instr->alu.src.offset = fsrc->offset / 8;
4300         return 0;
4301 }
4302
4303 static int
4304 instr_alu_shl_translate(struct rte_swx_pipeline *p,
4305                         struct action *action,
4306                         char **tokens,
4307                         int n_tokens,
4308                         struct instruction *instr,
4309                         struct instruction_data *data __rte_unused)
4310 {
4311         char *dst = tokens[1], *src = tokens[2];
4312         struct field *fdst, *fsrc;
4313         uint64_t src_val;
4314         uint32_t dst_struct_id = 0, src_struct_id = 0;
4315
4316         CHECK(n_tokens == 3, EINVAL);
4317
4318         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4319         CHECK(fdst, EINVAL);
4320         CHECK(!fdst->var_size, EINVAL);
4321
4322         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
4323         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4324         if (fsrc) {
4325                 CHECK(!fsrc->var_size, EINVAL);
4326
4327                 instr->type = INSTR_ALU_SHL;
4328                 if (dst[0] == 'h' && src[0] != 'h')
4329                         instr->type = INSTR_ALU_SHL_HM;
4330                 if (dst[0] != 'h' && src[0] == 'h')
4331                         instr->type = INSTR_ALU_SHL_MH;
4332                 if (dst[0] == 'h' && src[0] == 'h')
4333                         instr->type = INSTR_ALU_SHL_HH;
4334
4335                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4336                 instr->alu.dst.n_bits = fdst->n_bits;
4337                 instr->alu.dst.offset = fdst->offset / 8;
4338                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4339                 instr->alu.src.n_bits = fsrc->n_bits;
4340                 instr->alu.src.offset = fsrc->offset / 8;
4341                 return 0;
4342         }
4343
4344         /* SHL_MI, SHL_HI. */
4345         src_val = strtoull(src, &src, 0);
4346         CHECK(!src[0], EINVAL);
4347
4348         instr->type = INSTR_ALU_SHL_MI;
4349         if (dst[0] == 'h')
4350                 instr->type = INSTR_ALU_SHL_HI;
4351
4352         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4353         instr->alu.dst.n_bits = fdst->n_bits;
4354         instr->alu.dst.offset = fdst->offset / 8;
4355         instr->alu.src_val = src_val;
4356         return 0;
4357 }
4358
4359 static int
4360 instr_alu_shr_translate(struct rte_swx_pipeline *p,
4361                         struct action *action,
4362                         char **tokens,
4363                         int n_tokens,
4364                         struct instruction *instr,
4365                         struct instruction_data *data __rte_unused)
4366 {
4367         char *dst = tokens[1], *src = tokens[2];
4368         struct field *fdst, *fsrc;
4369         uint64_t src_val;
4370         uint32_t dst_struct_id = 0, src_struct_id = 0;
4371
4372         CHECK(n_tokens == 3, EINVAL);
4373
4374         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4375         CHECK(fdst, EINVAL);
4376         CHECK(!fdst->var_size, EINVAL);
4377
4378         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
4379         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4380         if (fsrc) {
4381                 CHECK(!fsrc->var_size, EINVAL);
4382
4383                 instr->type = INSTR_ALU_SHR;
4384                 if (dst[0] == 'h' && src[0] != 'h')
4385                         instr->type = INSTR_ALU_SHR_HM;
4386                 if (dst[0] != 'h' && src[0] == 'h')
4387                         instr->type = INSTR_ALU_SHR_MH;
4388                 if (dst[0] == 'h' && src[0] == 'h')
4389                         instr->type = INSTR_ALU_SHR_HH;
4390
4391                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4392                 instr->alu.dst.n_bits = fdst->n_bits;
4393                 instr->alu.dst.offset = fdst->offset / 8;
4394                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4395                 instr->alu.src.n_bits = fsrc->n_bits;
4396                 instr->alu.src.offset = fsrc->offset / 8;
4397                 return 0;
4398         }
4399
4400         /* SHR_MI, SHR_HI. */
4401         src_val = strtoull(src, &src, 0);
4402         CHECK(!src[0], EINVAL);
4403
4404         instr->type = INSTR_ALU_SHR_MI;
4405         if (dst[0] == 'h')
4406                 instr->type = INSTR_ALU_SHR_HI;
4407
4408         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4409         instr->alu.dst.n_bits = fdst->n_bits;
4410         instr->alu.dst.offset = fdst->offset / 8;
4411         instr->alu.src_val = src_val;
4412         return 0;
4413 }
4414
4415 static int
4416 instr_alu_and_translate(struct rte_swx_pipeline *p,
4417                         struct action *action,
4418                         char **tokens,
4419                         int n_tokens,
4420                         struct instruction *instr,
4421                         struct instruction_data *data __rte_unused)
4422 {
4423         char *dst = tokens[1], *src = tokens[2];
4424         struct field *fdst, *fsrc;
4425         uint64_t src_val;
4426         uint32_t dst_struct_id = 0, src_struct_id = 0;
4427
4428         CHECK(n_tokens == 3, EINVAL);
4429
4430         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4431         CHECK(fdst, EINVAL);
4432         CHECK(!fdst->var_size, EINVAL);
4433
4434         /* AND, AND_MH, AND_HM, AND_HH. */
4435         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4436         if (fsrc) {
4437                 CHECK(!fsrc->var_size, EINVAL);
4438
4439                 instr->type = INSTR_ALU_AND;
4440                 if (dst[0] != 'h' && src[0] == 'h')
4441                         instr->type = INSTR_ALU_AND_MH;
4442                 if (dst[0] == 'h' && src[0] != 'h')
4443                         instr->type = INSTR_ALU_AND_HM;
4444                 if (dst[0] == 'h' && src[0] == 'h')
4445                         instr->type = INSTR_ALU_AND_HH;
4446
4447                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4448                 instr->alu.dst.n_bits = fdst->n_bits;
4449                 instr->alu.dst.offset = fdst->offset / 8;
4450                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4451                 instr->alu.src.n_bits = fsrc->n_bits;
4452                 instr->alu.src.offset = fsrc->offset / 8;
4453                 return 0;
4454         }
4455
4456         /* AND_I. */
4457         src_val = strtoull(src, &src, 0);
4458         CHECK(!src[0], EINVAL);
4459
4460         if (dst[0] == 'h')
4461                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4462
4463         instr->type = INSTR_ALU_AND_I;
4464         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4465         instr->alu.dst.n_bits = fdst->n_bits;
4466         instr->alu.dst.offset = fdst->offset / 8;
4467         instr->alu.src_val = src_val;
4468         return 0;
4469 }
4470
4471 static int
4472 instr_alu_or_translate(struct rte_swx_pipeline *p,
4473                        struct action *action,
4474                        char **tokens,
4475                        int n_tokens,
4476                        struct instruction *instr,
4477                        struct instruction_data *data __rte_unused)
4478 {
4479         char *dst = tokens[1], *src = tokens[2];
4480         struct field *fdst, *fsrc;
4481         uint64_t src_val;
4482         uint32_t dst_struct_id = 0, src_struct_id = 0;
4483
4484         CHECK(n_tokens == 3, EINVAL);
4485
4486         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4487         CHECK(fdst, EINVAL);
4488         CHECK(!fdst->var_size, EINVAL);
4489
4490         /* OR, OR_MH, OR_HM, OR_HH. */
4491         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4492         if (fsrc) {
4493                 CHECK(!fsrc->var_size, EINVAL);
4494
4495                 instr->type = INSTR_ALU_OR;
4496                 if (dst[0] != 'h' && src[0] == 'h')
4497                         instr->type = INSTR_ALU_OR_MH;
4498                 if (dst[0] == 'h' && src[0] != 'h')
4499                         instr->type = INSTR_ALU_OR_HM;
4500                 if (dst[0] == 'h' && src[0] == 'h')
4501                         instr->type = INSTR_ALU_OR_HH;
4502
4503                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4504                 instr->alu.dst.n_bits = fdst->n_bits;
4505                 instr->alu.dst.offset = fdst->offset / 8;
4506                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4507                 instr->alu.src.n_bits = fsrc->n_bits;
4508                 instr->alu.src.offset = fsrc->offset / 8;
4509                 return 0;
4510         }
4511
4512         /* OR_I. */
4513         src_val = strtoull(src, &src, 0);
4514         CHECK(!src[0], EINVAL);
4515
4516         if (dst[0] == 'h')
4517                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4518
4519         instr->type = INSTR_ALU_OR_I;
4520         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4521         instr->alu.dst.n_bits = fdst->n_bits;
4522         instr->alu.dst.offset = fdst->offset / 8;
4523         instr->alu.src_val = src_val;
4524         return 0;
4525 }
4526
4527 static int
4528 instr_alu_xor_translate(struct rte_swx_pipeline *p,
4529                         struct action *action,
4530                         char **tokens,
4531                         int n_tokens,
4532                         struct instruction *instr,
4533                         struct instruction_data *data __rte_unused)
4534 {
4535         char *dst = tokens[1], *src = tokens[2];
4536         struct field *fdst, *fsrc;
4537         uint64_t src_val;
4538         uint32_t dst_struct_id = 0, src_struct_id = 0;
4539
4540         CHECK(n_tokens == 3, EINVAL);
4541
4542         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4543         CHECK(fdst, EINVAL);
4544         CHECK(!fdst->var_size, EINVAL);
4545
4546         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
4547         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4548         if (fsrc) {
4549                 CHECK(!fsrc->var_size, EINVAL);
4550
4551                 instr->type = INSTR_ALU_XOR;
4552                 if (dst[0] != 'h' && src[0] == 'h')
4553                         instr->type = INSTR_ALU_XOR_MH;
4554                 if (dst[0] == 'h' && src[0] != 'h')
4555                         instr->type = INSTR_ALU_XOR_HM;
4556                 if (dst[0] == 'h' && src[0] == 'h')
4557                         instr->type = INSTR_ALU_XOR_HH;
4558
4559                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4560                 instr->alu.dst.n_bits = fdst->n_bits;
4561                 instr->alu.dst.offset = fdst->offset / 8;
4562                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4563                 instr->alu.src.n_bits = fsrc->n_bits;
4564                 instr->alu.src.offset = fsrc->offset / 8;
4565                 return 0;
4566         }
4567
4568         /* XOR_I. */
4569         src_val = strtoull(src, &src, 0);
4570         CHECK(!src[0], EINVAL);
4571
4572         if (dst[0] == 'h')
4573                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4574
4575         instr->type = INSTR_ALU_XOR_I;
4576         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4577         instr->alu.dst.n_bits = fdst->n_bits;
4578         instr->alu.dst.offset = fdst->offset / 8;
4579         instr->alu.src_val = src_val;
4580         return 0;
4581 }
4582
4583 static inline void
4584 instr_alu_add_exec(struct rte_swx_pipeline *p)
4585 {
4586         struct thread *t = &p->threads[p->thread_id];
4587         struct instruction *ip = t->ip;
4588
4589         TRACE("[Thread %2u] add\n", p->thread_id);
4590
4591         /* Structs. */
4592         ALU(t, ip, +);
4593
4594         /* Thread. */
4595         thread_ip_inc(p);
4596 }
4597
4598 static inline void
4599 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
4600 {
4601         struct thread *t = &p->threads[p->thread_id];
4602         struct instruction *ip = t->ip;
4603
4604         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
4605
4606         /* Structs. */
4607         ALU_MH(t, ip, +);
4608
4609         /* Thread. */
4610         thread_ip_inc(p);
4611 }
4612
4613 static inline void
4614 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
4615 {
4616         struct thread *t = &p->threads[p->thread_id];
4617         struct instruction *ip = t->ip;
4618
4619         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
4620
4621         /* Structs. */
4622         ALU_HM(t, ip, +);
4623
4624         /* Thread. */
4625         thread_ip_inc(p);
4626 }
4627
4628 static inline void
4629 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
4630 {
4631         struct thread *t = &p->threads[p->thread_id];
4632         struct instruction *ip = t->ip;
4633
4634         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
4635
4636         /* Structs. */
4637         ALU_HH(t, ip, +);
4638
4639         /* Thread. */
4640         thread_ip_inc(p);
4641 }
4642
4643 static inline void
4644 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
4645 {
4646         struct thread *t = &p->threads[p->thread_id];
4647         struct instruction *ip = t->ip;
4648
4649         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
4650
4651         /* Structs. */
4652         ALU_MI(t, ip, +);
4653
4654         /* Thread. */
4655         thread_ip_inc(p);
4656 }
4657
4658 static inline void
4659 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
4660 {
4661         struct thread *t = &p->threads[p->thread_id];
4662         struct instruction *ip = t->ip;
4663
4664         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
4665
4666         /* Structs. */
4667         ALU_HI(t, ip, +);
4668
4669         /* Thread. */
4670         thread_ip_inc(p);
4671 }
4672
4673 static inline void
4674 instr_alu_sub_exec(struct rte_swx_pipeline *p)
4675 {
4676         struct thread *t = &p->threads[p->thread_id];
4677         struct instruction *ip = t->ip;
4678
4679         TRACE("[Thread %2u] sub\n", p->thread_id);
4680
4681         /* Structs. */
4682         ALU(t, ip, -);
4683
4684         /* Thread. */
4685         thread_ip_inc(p);
4686 }
4687
4688 static inline void
4689 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
4690 {
4691         struct thread *t = &p->threads[p->thread_id];
4692         struct instruction *ip = t->ip;
4693
4694         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
4695
4696         /* Structs. */
4697         ALU_MH(t, ip, -);
4698
4699         /* Thread. */
4700         thread_ip_inc(p);
4701 }
4702
4703 static inline void
4704 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4705 {
4706         struct thread *t = &p->threads[p->thread_id];
4707         struct instruction *ip = t->ip;
4708
4709         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4710
4711         /* Structs. */
4712         ALU_HM(t, ip, -);
4713
4714         /* Thread. */
4715         thread_ip_inc(p);
4716 }
4717
4718 static inline void
4719 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4720 {
4721         struct thread *t = &p->threads[p->thread_id];
4722         struct instruction *ip = t->ip;
4723
4724         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4725
4726         /* Structs. */
4727         ALU_HH(t, ip, -);
4728
4729         /* Thread. */
4730         thread_ip_inc(p);
4731 }
4732
4733 static inline void
4734 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4735 {
4736         struct thread *t = &p->threads[p->thread_id];
4737         struct instruction *ip = t->ip;
4738
4739         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4740
4741         /* Structs. */
4742         ALU_MI(t, ip, -);
4743
4744         /* Thread. */
4745         thread_ip_inc(p);
4746 }
4747
4748 static inline void
4749 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4750 {
4751         struct thread *t = &p->threads[p->thread_id];
4752         struct instruction *ip = t->ip;
4753
4754         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4755
4756         /* Structs. */
4757         ALU_HI(t, ip, -);
4758
4759         /* Thread. */
4760         thread_ip_inc(p);
4761 }
4762
4763 static inline void
4764 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4765 {
4766         struct thread *t = &p->threads[p->thread_id];
4767         struct instruction *ip = t->ip;
4768
4769         TRACE("[Thread %2u] shl\n", p->thread_id);
4770
4771         /* Structs. */
4772         ALU(t, ip, <<);
4773
4774         /* Thread. */
4775         thread_ip_inc(p);
4776 }
4777
4778 static inline void
4779 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4780 {
4781         struct thread *t = &p->threads[p->thread_id];
4782         struct instruction *ip = t->ip;
4783
4784         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4785
4786         /* Structs. */
4787         ALU_MH(t, ip, <<);
4788
4789         /* Thread. */
4790         thread_ip_inc(p);
4791 }
4792
4793 static inline void
4794 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4795 {
4796         struct thread *t = &p->threads[p->thread_id];
4797         struct instruction *ip = t->ip;
4798
4799         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4800
4801         /* Structs. */
4802         ALU_HM(t, ip, <<);
4803
4804         /* Thread. */
4805         thread_ip_inc(p);
4806 }
4807
4808 static inline void
4809 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4810 {
4811         struct thread *t = &p->threads[p->thread_id];
4812         struct instruction *ip = t->ip;
4813
4814         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4815
4816         /* Structs. */
4817         ALU_HH(t, ip, <<);
4818
4819         /* Thread. */
4820         thread_ip_inc(p);
4821 }
4822
4823 static inline void
4824 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4825 {
4826         struct thread *t = &p->threads[p->thread_id];
4827         struct instruction *ip = t->ip;
4828
4829         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4830
4831         /* Structs. */
4832         ALU_MI(t, ip, <<);
4833
4834         /* Thread. */
4835         thread_ip_inc(p);
4836 }
4837
4838 static inline void
4839 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4840 {
4841         struct thread *t = &p->threads[p->thread_id];
4842         struct instruction *ip = t->ip;
4843
4844         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4845
4846         /* Structs. */
4847         ALU_HI(t, ip, <<);
4848
4849         /* Thread. */
4850         thread_ip_inc(p);
4851 }
4852
4853 static inline void
4854 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4855 {
4856         struct thread *t = &p->threads[p->thread_id];
4857         struct instruction *ip = t->ip;
4858
4859         TRACE("[Thread %2u] shr\n", p->thread_id);
4860
4861         /* Structs. */
4862         ALU(t, ip, >>);
4863
4864         /* Thread. */
4865         thread_ip_inc(p);
4866 }
4867
4868 static inline void
4869 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4870 {
4871         struct thread *t = &p->threads[p->thread_id];
4872         struct instruction *ip = t->ip;
4873
4874         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4875
4876         /* Structs. */
4877         ALU_MH(t, ip, >>);
4878
4879         /* Thread. */
4880         thread_ip_inc(p);
4881 }
4882
4883 static inline void
4884 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4885 {
4886         struct thread *t = &p->threads[p->thread_id];
4887         struct instruction *ip = t->ip;
4888
4889         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4890
4891         /* Structs. */
4892         ALU_HM(t, ip, >>);
4893
4894         /* Thread. */
4895         thread_ip_inc(p);
4896 }
4897
4898 static inline void
4899 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4900 {
4901         struct thread *t = &p->threads[p->thread_id];
4902         struct instruction *ip = t->ip;
4903
4904         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4905
4906         /* Structs. */
4907         ALU_HH(t, ip, >>);
4908
4909         /* Thread. */
4910         thread_ip_inc(p);
4911 }
4912
4913 static inline void
4914 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4915 {
4916         struct thread *t = &p->threads[p->thread_id];
4917         struct instruction *ip = t->ip;
4918
4919         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4920
4921         /* Structs. */
4922         ALU_MI(t, ip, >>);
4923
4924         /* Thread. */
4925         thread_ip_inc(p);
4926 }
4927
4928 static inline void
4929 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4930 {
4931         struct thread *t = &p->threads[p->thread_id];
4932         struct instruction *ip = t->ip;
4933
4934         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4935
4936         /* Structs. */
4937         ALU_HI(t, ip, >>);
4938
4939         /* Thread. */
4940         thread_ip_inc(p);
4941 }
4942
4943 static inline void
4944 instr_alu_and_exec(struct rte_swx_pipeline *p)
4945 {
4946         struct thread *t = &p->threads[p->thread_id];
4947         struct instruction *ip = t->ip;
4948
4949         TRACE("[Thread %2u] and\n", p->thread_id);
4950
4951         /* Structs. */
4952         ALU(t, ip, &);
4953
4954         /* Thread. */
4955         thread_ip_inc(p);
4956 }
4957
4958 static inline void
4959 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
4960 {
4961         struct thread *t = &p->threads[p->thread_id];
4962         struct instruction *ip = t->ip;
4963
4964         TRACE("[Thread %2u] and (mh)\n", p->thread_id);
4965
4966         /* Structs. */
4967         ALU_MH(t, ip, &);
4968
4969         /* Thread. */
4970         thread_ip_inc(p);
4971 }
4972
4973 static inline void
4974 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
4975 {
4976         struct thread *t = &p->threads[p->thread_id];
4977         struct instruction *ip = t->ip;
4978
4979         TRACE("[Thread %2u] and (hm)\n", p->thread_id);
4980
4981         /* Structs. */
4982         ALU_HM_FAST(t, ip, &);
4983
4984         /* Thread. */
4985         thread_ip_inc(p);
4986 }
4987
4988 static inline void
4989 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
4990 {
4991         struct thread *t = &p->threads[p->thread_id];
4992         struct instruction *ip = t->ip;
4993
4994         TRACE("[Thread %2u] and (hh)\n", p->thread_id);
4995
4996         /* Structs. */
4997         ALU_HH_FAST(t, ip, &);
4998
4999         /* Thread. */
5000         thread_ip_inc(p);
5001 }
5002
5003 static inline void
5004 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
5005 {
5006         struct thread *t = &p->threads[p->thread_id];
5007         struct instruction *ip = t->ip;
5008
5009         TRACE("[Thread %2u] and (i)\n", p->thread_id);
5010
5011         /* Structs. */
5012         ALU_I(t, ip, &);
5013
5014         /* Thread. */
5015         thread_ip_inc(p);
5016 }
5017
5018 static inline void
5019 instr_alu_or_exec(struct rte_swx_pipeline *p)
5020 {
5021         struct thread *t = &p->threads[p->thread_id];
5022         struct instruction *ip = t->ip;
5023
5024         TRACE("[Thread %2u] or\n", p->thread_id);
5025
5026         /* Structs. */
5027         ALU(t, ip, |);
5028
5029         /* Thread. */
5030         thread_ip_inc(p);
5031 }
5032
5033 static inline void
5034 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
5035 {
5036         struct thread *t = &p->threads[p->thread_id];
5037         struct instruction *ip = t->ip;
5038
5039         TRACE("[Thread %2u] or (mh)\n", p->thread_id);
5040
5041         /* Structs. */
5042         ALU_MH(t, ip, |);
5043
5044         /* Thread. */
5045         thread_ip_inc(p);
5046 }
5047
5048 static inline void
5049 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
5050 {
5051         struct thread *t = &p->threads[p->thread_id];
5052         struct instruction *ip = t->ip;
5053
5054         TRACE("[Thread %2u] or (hm)\n", p->thread_id);
5055
5056         /* Structs. */
5057         ALU_HM_FAST(t, ip, |);
5058
5059         /* Thread. */
5060         thread_ip_inc(p);
5061 }
5062
5063 static inline void
5064 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
5065 {
5066         struct thread *t = &p->threads[p->thread_id];
5067         struct instruction *ip = t->ip;
5068
5069         TRACE("[Thread %2u] or (hh)\n", p->thread_id);
5070
5071         /* Structs. */
5072         ALU_HH_FAST(t, ip, |);
5073
5074         /* Thread. */
5075         thread_ip_inc(p);
5076 }
5077
5078 static inline void
5079 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
5080 {
5081         struct thread *t = &p->threads[p->thread_id];
5082         struct instruction *ip = t->ip;
5083
5084         TRACE("[Thread %2u] or (i)\n", p->thread_id);
5085
5086         /* Structs. */
5087         ALU_I(t, ip, |);
5088
5089         /* Thread. */
5090         thread_ip_inc(p);
5091 }
5092
5093 static inline void
5094 instr_alu_xor_exec(struct rte_swx_pipeline *p)
5095 {
5096         struct thread *t = &p->threads[p->thread_id];
5097         struct instruction *ip = t->ip;
5098
5099         TRACE("[Thread %2u] xor\n", p->thread_id);
5100
5101         /* Structs. */
5102         ALU(t, ip, ^);
5103
5104         /* Thread. */
5105         thread_ip_inc(p);
5106 }
5107
5108 static inline void
5109 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
5110 {
5111         struct thread *t = &p->threads[p->thread_id];
5112         struct instruction *ip = t->ip;
5113
5114         TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
5115
5116         /* Structs. */
5117         ALU_MH(t, ip, ^);
5118
5119         /* Thread. */
5120         thread_ip_inc(p);
5121 }
5122
5123 static inline void
5124 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
5125 {
5126         struct thread *t = &p->threads[p->thread_id];
5127         struct instruction *ip = t->ip;
5128
5129         TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
5130
5131         /* Structs. */
5132         ALU_HM_FAST(t, ip, ^);
5133
5134         /* Thread. */
5135         thread_ip_inc(p);
5136 }
5137
5138 static inline void
5139 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
5140 {
5141         struct thread *t = &p->threads[p->thread_id];
5142         struct instruction *ip = t->ip;
5143
5144         TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
5145
5146         /* Structs. */
5147         ALU_HH_FAST(t, ip, ^);
5148
5149         /* Thread. */
5150         thread_ip_inc(p);
5151 }
5152
5153 static inline void
5154 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
5155 {
5156         struct thread *t = &p->threads[p->thread_id];
5157         struct instruction *ip = t->ip;
5158
5159         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
5160
5161         /* Structs. */
5162         ALU_I(t, ip, ^);
5163
5164         /* Thread. */
5165         thread_ip_inc(p);
5166 }
5167
5168 static inline void
5169 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
5170 {
5171         struct thread *t = &p->threads[p->thread_id];
5172         struct instruction *ip = t->ip;
5173         uint8_t *dst_struct, *src_struct;
5174         uint16_t *dst16_ptr, dst;
5175         uint64_t *src64_ptr, src64, src64_mask, src;
5176         uint64_t r;
5177
5178         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
5179
5180         /* Structs. */
5181         dst_struct = t->structs[ip->alu.dst.struct_id];
5182         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5183         dst = *dst16_ptr;
5184
5185         src_struct = t->structs[ip->alu.src.struct_id];
5186         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
5187         src64 = *src64_ptr;
5188         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
5189         src = src64 & src64_mask;
5190
5191         r = dst;
5192         r = ~r & 0xFFFF;
5193
5194         /* The first input (r) is a 16-bit number. The second and the third
5195          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
5196          * three numbers (output r) is a 34-bit number.
5197          */
5198         r += (src >> 32) + (src & 0xFFFFFFFF);
5199
5200         /* The first input is a 16-bit number. The second input is an 18-bit
5201          * number. In the worst case scenario, the sum of the two numbers is a
5202          * 19-bit number.
5203          */
5204         r = (r & 0xFFFF) + (r >> 16);
5205
5206         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5207          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
5208          */
5209         r = (r & 0xFFFF) + (r >> 16);
5210
5211         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5212          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5213          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
5214          * therefore the output r is always a 16-bit number.
5215          */
5216         r = (r & 0xFFFF) + (r >> 16);
5217
5218         r = ~r & 0xFFFF;
5219         r = r ? r : 0xFFFF;
5220
5221         *dst16_ptr = (uint16_t)r;
5222
5223         /* Thread. */
5224         thread_ip_inc(p);
5225 }
5226
5227 static inline void
5228 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
5229 {
5230         struct thread *t = &p->threads[p->thread_id];
5231         struct instruction *ip = t->ip;
5232         uint8_t *dst_struct, *src_struct;
5233         uint16_t *dst16_ptr, dst;
5234         uint64_t *src64_ptr, src64, src64_mask, src;
5235         uint64_t r;
5236
5237         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
5238
5239         /* Structs. */
5240         dst_struct = t->structs[ip->alu.dst.struct_id];
5241         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5242         dst = *dst16_ptr;
5243
5244         src_struct = t->structs[ip->alu.src.struct_id];
5245         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
5246         src64 = *src64_ptr;
5247         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
5248         src = src64 & src64_mask;
5249
5250         r = dst;
5251         r = ~r & 0xFFFF;
5252
5253         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
5254          * the following sequence of operations in 2's complement arithmetic:
5255          *    a '- b = (a - b) % 0xFFFF.
5256          *
5257          * In order to prevent an underflow for the below subtraction, in which
5258          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
5259          * minuend), we first add a multiple of the 0xFFFF modulus to the
5260          * minuend. The number we add to the minuend needs to be a 34-bit number
5261          * or higher, so for readability reasons we picked the 36-bit multiple.
5262          * We are effectively turning the 16-bit minuend into a 36-bit number:
5263          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
5264          */
5265         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
5266
5267         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
5268          * result (the output r) is a 36-bit number.
5269          */
5270         r -= (src >> 32) + (src & 0xFFFFFFFF);
5271
5272         /* The first input is a 16-bit number. The second input is a 20-bit
5273          * number. Their sum is a 21-bit number.
5274          */
5275         r = (r & 0xFFFF) + (r >> 16);
5276
5277         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5278          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
5279          */
5280         r = (r & 0xFFFF) + (r >> 16);
5281
5282         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5283          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5284          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5285          * generated, therefore the output r is always a 16-bit number.
5286          */
5287         r = (r & 0xFFFF) + (r >> 16);
5288
5289         r = ~r & 0xFFFF;
5290         r = r ? r : 0xFFFF;
5291
5292         *dst16_ptr = (uint16_t)r;
5293
5294         /* Thread. */
5295         thread_ip_inc(p);
5296 }
5297
5298 static inline void
5299 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
5300 {
5301         struct thread *t = &p->threads[p->thread_id];
5302         struct instruction *ip = t->ip;
5303         uint8_t *dst_struct, *src_struct;
5304         uint16_t *dst16_ptr;
5305         uint32_t *src32_ptr;
5306         uint64_t r0, r1;
5307
5308         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
5309
5310         /* Structs. */
5311         dst_struct = t->structs[ip->alu.dst.struct_id];
5312         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5313
5314         src_struct = t->structs[ip->alu.src.struct_id];
5315         src32_ptr = (uint32_t *)&src_struct[0];
5316
5317         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
5318         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
5319         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
5320         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
5321         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
5322
5323         /* The first input is a 16-bit number. The second input is a 19-bit
5324          * number. Their sum is a 20-bit number.
5325          */
5326         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5327
5328         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5329          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
5330          */
5331         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5332
5333         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5334          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5335          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
5336          * generated, therefore the output r is always a 16-bit number.
5337          */
5338         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5339
5340         r0 = ~r0 & 0xFFFF;
5341         r0 = r0 ? r0 : 0xFFFF;
5342
5343         *dst16_ptr = (uint16_t)r0;
5344
5345         /* Thread. */
5346         thread_ip_inc(p);
5347 }
5348
5349 static inline void
5350 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
5351 {
5352         struct thread *t = &p->threads[p->thread_id];
5353         struct instruction *ip = t->ip;
5354         uint8_t *dst_struct, *src_struct;
5355         uint16_t *dst16_ptr;
5356         uint32_t *src32_ptr;
5357         uint64_t r = 0;
5358         uint32_t i;
5359
5360         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
5361
5362         /* Structs. */
5363         dst_struct = t->structs[ip->alu.dst.struct_id];
5364         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5365
5366         src_struct = t->structs[ip->alu.src.struct_id];
5367         src32_ptr = (uint32_t *)&src_struct[0];
5368
5369         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
5370          * Therefore, in the worst case scenario, a 35-bit number is added to a
5371          * 16-bit number (the input r), so the output r is 36-bit number.
5372          */
5373         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
5374                 r += *src32_ptr;
5375
5376         /* The first input is a 16-bit number. The second input is a 20-bit
5377          * number. Their sum is a 21-bit number.
5378          */
5379         r = (r & 0xFFFF) + (r >> 16);
5380
5381         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5382          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
5383          */
5384         r = (r & 0xFFFF) + (r >> 16);
5385
5386         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5387          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5388          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5389          * generated, therefore the output r is always a 16-bit number.
5390          */
5391         r = (r & 0xFFFF) + (r >> 16);
5392
5393         r = ~r & 0xFFFF;
5394         r = r ? r : 0xFFFF;
5395
5396         *dst16_ptr = (uint16_t)r;
5397
5398         /* Thread. */
5399         thread_ip_inc(p);
5400 }
5401
5402 /*
5403  * Register array.
5404  */
5405 static struct regarray *
5406 regarray_find(struct rte_swx_pipeline *p, const char *name);
5407
5408 static int
5409 instr_regprefetch_translate(struct rte_swx_pipeline *p,
5410                       struct action *action,
5411                       char **tokens,
5412                       int n_tokens,
5413                       struct instruction *instr,
5414                       struct instruction_data *data __rte_unused)
5415 {
5416         char *regarray = tokens[1], *idx = tokens[2];
5417         struct regarray *r;
5418         struct field *fidx;
5419         uint32_t idx_struct_id, idx_val;
5420
5421         CHECK(n_tokens == 3, EINVAL);
5422
5423         r = regarray_find(p, regarray);
5424         CHECK(r, EINVAL);
5425
5426         /* REGPREFETCH_RH, REGPREFETCH_RM. */
5427         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5428         if (fidx) {
5429                 CHECK(!fidx->var_size, EINVAL);
5430
5431                 instr->type = INSTR_REGPREFETCH_RM;
5432                 if (idx[0] == 'h')
5433                         instr->type = INSTR_REGPREFETCH_RH;
5434
5435                 instr->regarray.regarray_id = r->id;
5436                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5437                 instr->regarray.idx.n_bits = fidx->n_bits;
5438                 instr->regarray.idx.offset = fidx->offset / 8;
5439                 instr->regarray.dstsrc_val = 0; /* Unused. */
5440                 return 0;
5441         }
5442
5443         /* REGPREFETCH_RI. */
5444         idx_val = strtoul(idx, &idx, 0);
5445         CHECK(!idx[0], EINVAL);
5446
5447         instr->type = INSTR_REGPREFETCH_RI;
5448         instr->regarray.regarray_id = r->id;
5449         instr->regarray.idx_val = idx_val;
5450         instr->regarray.dstsrc_val = 0; /* Unused. */
5451         return 0;
5452 }
5453
5454 static int
5455 instr_regrd_translate(struct rte_swx_pipeline *p,
5456                       struct action *action,
5457                       char **tokens,
5458                       int n_tokens,
5459                       struct instruction *instr,
5460                       struct instruction_data *data __rte_unused)
5461 {
5462         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
5463         struct regarray *r;
5464         struct field *fdst, *fidx;
5465         uint32_t dst_struct_id, idx_struct_id, idx_val;
5466
5467         CHECK(n_tokens == 4, EINVAL);
5468
5469         r = regarray_find(p, regarray);
5470         CHECK(r, EINVAL);
5471
5472         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
5473         CHECK(fdst, EINVAL);
5474         CHECK(!fdst->var_size, EINVAL);
5475
5476         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
5477         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5478         if (fidx) {
5479                 CHECK(!fidx->var_size, EINVAL);
5480
5481                 instr->type = INSTR_REGRD_MRM;
5482                 if (dst[0] == 'h' && idx[0] != 'h')
5483                         instr->type = INSTR_REGRD_HRM;
5484                 if (dst[0] != 'h' && idx[0] == 'h')
5485                         instr->type = INSTR_REGRD_MRH;
5486                 if (dst[0] == 'h' && idx[0] == 'h')
5487                         instr->type = INSTR_REGRD_HRH;
5488
5489                 instr->regarray.regarray_id = r->id;
5490                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5491                 instr->regarray.idx.n_bits = fidx->n_bits;
5492                 instr->regarray.idx.offset = fidx->offset / 8;
5493                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5494                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
5495                 instr->regarray.dstsrc.offset = fdst->offset / 8;
5496                 return 0;
5497         }
5498
5499         /* REGRD_MRI, REGRD_HRI. */
5500         idx_val = strtoul(idx, &idx, 0);
5501         CHECK(!idx[0], EINVAL);
5502
5503         instr->type = INSTR_REGRD_MRI;
5504         if (dst[0] == 'h')
5505                 instr->type = INSTR_REGRD_HRI;
5506
5507         instr->regarray.regarray_id = r->id;
5508         instr->regarray.idx_val = idx_val;
5509         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5510         instr->regarray.dstsrc.n_bits = fdst->n_bits;
5511         instr->regarray.dstsrc.offset = fdst->offset / 8;
5512         return 0;
5513 }
5514
5515 static int
5516 instr_regwr_translate(struct rte_swx_pipeline *p,
5517                       struct action *action,
5518                       char **tokens,
5519                       int n_tokens,
5520                       struct instruction *instr,
5521                       struct instruction_data *data __rte_unused)
5522 {
5523         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5524         struct regarray *r;
5525         struct field *fidx, *fsrc;
5526         uint64_t src_val;
5527         uint32_t idx_struct_id, idx_val, src_struct_id;
5528
5529         CHECK(n_tokens == 4, EINVAL);
5530
5531         r = regarray_find(p, regarray);
5532         CHECK(r, EINVAL);
5533
5534         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
5535         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5536         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5537         if (fidx && fsrc) {
5538                 CHECK(!fidx->var_size, EINVAL);
5539                 CHECK(!fsrc->var_size, EINVAL);
5540
5541                 instr->type = INSTR_REGWR_RMM;
5542                 if (idx[0] == 'h' && src[0] != 'h')
5543                         instr->type = INSTR_REGWR_RHM;
5544                 if (idx[0] != 'h' && src[0] == 'h')
5545                         instr->type = INSTR_REGWR_RMH;
5546                 if (idx[0] == 'h' && src[0] == 'h')
5547                         instr->type = INSTR_REGWR_RHH;
5548
5549                 instr->regarray.regarray_id = r->id;
5550                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5551                 instr->regarray.idx.n_bits = fidx->n_bits;
5552                 instr->regarray.idx.offset = fidx->offset / 8;
5553                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5554                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5555                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5556                 return 0;
5557         }
5558
5559         /* REGWR_RHI, REGWR_RMI. */
5560         if (fidx && !fsrc) {
5561                 CHECK(!fidx->var_size, EINVAL);
5562
5563                 src_val = strtoull(src, &src, 0);
5564                 CHECK(!src[0], EINVAL);
5565
5566                 instr->type = INSTR_REGWR_RMI;
5567                 if (idx[0] == 'h')
5568                         instr->type = INSTR_REGWR_RHI;
5569
5570                 instr->regarray.regarray_id = r->id;
5571                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5572                 instr->regarray.idx.n_bits = fidx->n_bits;
5573                 instr->regarray.idx.offset = fidx->offset / 8;
5574                 instr->regarray.dstsrc_val = src_val;
5575                 return 0;
5576         }
5577
5578         /* REGWR_RIH, REGWR_RIM. */
5579         if (!fidx && fsrc) {
5580                 idx_val = strtoul(idx, &idx, 0);
5581                 CHECK(!idx[0], EINVAL);
5582
5583                 CHECK(!fsrc->var_size, EINVAL);
5584
5585                 instr->type = INSTR_REGWR_RIM;
5586                 if (src[0] == 'h')
5587                         instr->type = INSTR_REGWR_RIH;
5588
5589                 instr->regarray.regarray_id = r->id;
5590                 instr->regarray.idx_val = idx_val;
5591                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5592                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5593                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5594                 return 0;
5595         }
5596
5597         /* REGWR_RII. */
5598         src_val = strtoull(src, &src, 0);
5599         CHECK(!src[0], EINVAL);
5600
5601         idx_val = strtoul(idx, &idx, 0);
5602         CHECK(!idx[0], EINVAL);
5603
5604         instr->type = INSTR_REGWR_RII;
5605         instr->regarray.idx_val = idx_val;
5606         instr->regarray.dstsrc_val = src_val;
5607
5608         return 0;
5609 }
5610
5611 static int
5612 instr_regadd_translate(struct rte_swx_pipeline *p,
5613                        struct action *action,
5614                        char **tokens,
5615                        int n_tokens,
5616                        struct instruction *instr,
5617                        struct instruction_data *data __rte_unused)
5618 {
5619         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5620         struct regarray *r;
5621         struct field *fidx, *fsrc;
5622         uint64_t src_val;
5623         uint32_t idx_struct_id, idx_val, src_struct_id;
5624
5625         CHECK(n_tokens == 4, EINVAL);
5626
5627         r = regarray_find(p, regarray);
5628         CHECK(r, EINVAL);
5629
5630         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
5631         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5632         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5633         if (fidx && fsrc) {
5634                 CHECK(!fidx->var_size, EINVAL);
5635                 CHECK(!fsrc->var_size, EINVAL);
5636
5637                 instr->type = INSTR_REGADD_RMM;
5638                 if (idx[0] == 'h' && src[0] != 'h')
5639                         instr->type = INSTR_REGADD_RHM;
5640                 if (idx[0] != 'h' && src[0] == 'h')
5641                         instr->type = INSTR_REGADD_RMH;
5642                 if (idx[0] == 'h' && src[0] == 'h')
5643                         instr->type = INSTR_REGADD_RHH;
5644
5645                 instr->regarray.regarray_id = r->id;
5646                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5647                 instr->regarray.idx.n_bits = fidx->n_bits;
5648                 instr->regarray.idx.offset = fidx->offset / 8;
5649                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5650                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5651                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5652                 return 0;
5653         }
5654
5655         /* REGADD_RHI, REGADD_RMI. */
5656         if (fidx && !fsrc) {
5657                 CHECK(!fidx->var_size, EINVAL);
5658
5659                 src_val = strtoull(src, &src, 0);
5660                 CHECK(!src[0], EINVAL);
5661
5662                 instr->type = INSTR_REGADD_RMI;
5663                 if (idx[0] == 'h')
5664                         instr->type = INSTR_REGADD_RHI;
5665
5666                 instr->regarray.regarray_id = r->id;
5667                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5668                 instr->regarray.idx.n_bits = fidx->n_bits;
5669                 instr->regarray.idx.offset = fidx->offset / 8;
5670                 instr->regarray.dstsrc_val = src_val;
5671                 return 0;
5672         }
5673
5674         /* REGADD_RIH, REGADD_RIM. */
5675         if (!fidx && fsrc) {
5676                 idx_val = strtoul(idx, &idx, 0);
5677                 CHECK(!idx[0], EINVAL);
5678
5679                 CHECK(!fsrc->var_size, EINVAL);
5680
5681                 instr->type = INSTR_REGADD_RIM;
5682                 if (src[0] == 'h')
5683                         instr->type = INSTR_REGADD_RIH;
5684
5685                 instr->regarray.regarray_id = r->id;
5686                 instr->regarray.idx_val = idx_val;
5687                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5688                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5689                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5690                 return 0;
5691         }
5692
5693         /* REGADD_RII. */
5694         src_val = strtoull(src, &src, 0);
5695         CHECK(!src[0], EINVAL);
5696
5697         idx_val = strtoul(idx, &idx, 0);
5698         CHECK(!idx[0], EINVAL);
5699
5700         instr->type = INSTR_REGADD_RII;
5701         instr->regarray.idx_val = idx_val;
5702         instr->regarray.dstsrc_val = src_val;
5703         return 0;
5704 }
5705
5706 static inline uint64_t *
5707 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
5708 {
5709         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5710         return r->regarray;
5711 }
5712
5713 static inline uint64_t
5714 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5715 {
5716         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5717
5718         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5719         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5720         uint64_t idx64 = *idx64_ptr;
5721         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
5722         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5723
5724         return idx;
5725 }
5726
5727 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5728
5729 static inline uint64_t
5730 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5731 {
5732         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5733
5734         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5735         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5736         uint64_t idx64 = *idx64_ptr;
5737         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
5738
5739         return idx;
5740 }
5741
5742 #else
5743
5744 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
5745
5746 #endif
5747
5748 static inline uint64_t
5749 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5750 {
5751         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5752
5753         uint64_t idx = ip->regarray.idx_val & r->size_mask;
5754
5755         return idx;
5756 }
5757
5758 static inline uint64_t
5759 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
5760 {
5761         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5762         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5763         uint64_t src64 = *src64_ptr;
5764         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5765         uint64_t src = src64 & src64_mask;
5766
5767         return src;
5768 }
5769
5770 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5771
5772 static inline uint64_t
5773 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
5774 {
5775         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5776         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5777         uint64_t src64 = *src64_ptr;
5778         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
5779
5780         return src;
5781 }
5782
5783 #else
5784
5785 #define instr_regarray_src_nbo instr_regarray_src_hbo
5786
5787 #endif
5788
5789 static inline void
5790 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5791 {
5792         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5793         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5794         uint64_t dst64 = *dst64_ptr;
5795         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5796
5797         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5798
5799 }
5800
5801 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5802
5803 static inline void
5804 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5805 {
5806         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5807         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5808         uint64_t dst64 = *dst64_ptr;
5809         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5810
5811         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
5812         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5813 }
5814
5815 #else
5816
5817 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
5818
5819 #endif
5820
5821 static inline void
5822 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
5823 {
5824         struct thread *t = &p->threads[p->thread_id];
5825         struct instruction *ip = t->ip;
5826         uint64_t *regarray, idx;
5827
5828         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
5829
5830         /* Structs. */
5831         regarray = instr_regarray_regarray(p, ip);
5832         idx = instr_regarray_idx_nbo(p, t, ip);
5833         rte_prefetch0(&regarray[idx]);
5834
5835         /* Thread. */
5836         thread_ip_inc(p);
5837 }
5838
5839 static inline void
5840 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
5841 {
5842         struct thread *t = &p->threads[p->thread_id];
5843         struct instruction *ip = t->ip;
5844         uint64_t *regarray, idx;
5845
5846         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
5847
5848         /* Structs. */
5849         regarray = instr_regarray_regarray(p, ip);
5850         idx = instr_regarray_idx_hbo(p, t, ip);
5851         rte_prefetch0(&regarray[idx]);
5852
5853         /* Thread. */
5854         thread_ip_inc(p);
5855 }
5856
5857 static inline void
5858 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
5859 {
5860         struct thread *t = &p->threads[p->thread_id];
5861         struct instruction *ip = t->ip;
5862         uint64_t *regarray, idx;
5863
5864         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
5865
5866         /* Structs. */
5867         regarray = instr_regarray_regarray(p, ip);
5868         idx = instr_regarray_idx_imm(p, ip);
5869         rte_prefetch0(&regarray[idx]);
5870
5871         /* Thread. */
5872         thread_ip_inc(p);
5873 }
5874
5875 static inline void
5876 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
5877 {
5878         struct thread *t = &p->threads[p->thread_id];
5879         struct instruction *ip = t->ip;
5880         uint64_t *regarray, idx;
5881
5882         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
5883
5884         /* Structs. */
5885         regarray = instr_regarray_regarray(p, ip);
5886         idx = instr_regarray_idx_nbo(p, t, ip);
5887         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5888
5889         /* Thread. */
5890         thread_ip_inc(p);
5891 }
5892
5893 static inline void
5894 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
5895 {
5896         struct thread *t = &p->threads[p->thread_id];
5897         struct instruction *ip = t->ip;
5898         uint64_t *regarray, idx;
5899
5900         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
5901
5902         /* Structs. */
5903         regarray = instr_regarray_regarray(p, ip);
5904         idx = instr_regarray_idx_hbo(p, t, ip);
5905         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5906
5907         /* Thread. */
5908         thread_ip_inc(p);
5909 }
5910
5911 static inline void
5912 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
5913 {
5914         struct thread *t = &p->threads[p->thread_id];
5915         struct instruction *ip = t->ip;
5916         uint64_t *regarray, idx;
5917
5918         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
5919
5920         /* Structs. */
5921         regarray = instr_regarray_regarray(p, ip);
5922         idx = instr_regarray_idx_nbo(p, t, ip);
5923         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5924
5925         /* Thread. */
5926         thread_ip_inc(p);
5927 }
5928
5929 static inline void
5930 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
5931 {
5932         struct thread *t = &p->threads[p->thread_id];
5933         struct instruction *ip = t->ip;
5934         uint64_t *regarray, idx;
5935
5936         /* Structs. */
5937         regarray = instr_regarray_regarray(p, ip);
5938         idx = instr_regarray_idx_hbo(p, t, ip);
5939         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5940
5941         /* Thread. */
5942         thread_ip_inc(p);
5943 }
5944
5945 static inline void
5946 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
5947 {
5948         struct thread *t = &p->threads[p->thread_id];
5949         struct instruction *ip = t->ip;
5950         uint64_t *regarray, idx;
5951
5952         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
5953
5954         /* Structs. */
5955         regarray = instr_regarray_regarray(p, ip);
5956         idx = instr_regarray_idx_imm(p, ip);
5957         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5958
5959         /* Thread. */
5960         thread_ip_inc(p);
5961 }
5962
5963 static inline void
5964 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
5965 {
5966         struct thread *t = &p->threads[p->thread_id];
5967         struct instruction *ip = t->ip;
5968         uint64_t *regarray, idx;
5969
5970         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
5971
5972         /* Structs. */
5973         regarray = instr_regarray_regarray(p, ip);
5974         idx = instr_regarray_idx_imm(p, ip);
5975         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5976
5977         /* Thread. */
5978         thread_ip_inc(p);
5979 }
5980
5981 static inline void
5982 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
5983 {
5984         struct thread *t = &p->threads[p->thread_id];
5985         struct instruction *ip = t->ip;
5986         uint64_t *regarray, idx, src;
5987
5988         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
5989
5990         /* Structs. */
5991         regarray = instr_regarray_regarray(p, ip);
5992         idx = instr_regarray_idx_nbo(p, t, ip);
5993         src = instr_regarray_src_nbo(t, ip);
5994         regarray[idx] = src;
5995
5996         /* Thread. */
5997         thread_ip_inc(p);
5998 }
5999
6000 static inline void
6001 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
6002 {
6003         struct thread *t = &p->threads[p->thread_id];
6004         struct instruction *ip = t->ip;
6005         uint64_t *regarray, idx, src;
6006
6007         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
6008
6009         /* Structs. */
6010         regarray = instr_regarray_regarray(p, ip);
6011         idx = instr_regarray_idx_nbo(p, t, ip);
6012         src = instr_regarray_src_hbo(t, ip);
6013         regarray[idx] = src;
6014
6015         /* Thread. */
6016         thread_ip_inc(p);
6017 }
6018
6019 static inline void
6020 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
6021 {
6022         struct thread *t = &p->threads[p->thread_id];
6023         struct instruction *ip = t->ip;
6024         uint64_t *regarray, idx, src;
6025
6026         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
6027
6028         /* Structs. */
6029         regarray = instr_regarray_regarray(p, ip);
6030         idx = instr_regarray_idx_hbo(p, t, ip);
6031         src = instr_regarray_src_nbo(t, ip);
6032         regarray[idx] = src;
6033
6034         /* Thread. */
6035         thread_ip_inc(p);
6036 }
6037
6038 static inline void
6039 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
6040 {
6041         struct thread *t = &p->threads[p->thread_id];
6042         struct instruction *ip = t->ip;
6043         uint64_t *regarray, idx, src;
6044
6045         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
6046
6047         /* Structs. */
6048         regarray = instr_regarray_regarray(p, ip);
6049         idx = instr_regarray_idx_hbo(p, t, ip);
6050         src = instr_regarray_src_hbo(t, ip);
6051         regarray[idx] = src;
6052
6053         /* Thread. */
6054         thread_ip_inc(p);
6055 }
6056
6057 static inline void
6058 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
6059 {
6060         struct thread *t = &p->threads[p->thread_id];
6061         struct instruction *ip = t->ip;
6062         uint64_t *regarray, idx, src;
6063
6064         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
6065
6066         /* Structs. */
6067         regarray = instr_regarray_regarray(p, ip);
6068         idx = instr_regarray_idx_nbo(p, t, ip);
6069         src = ip->regarray.dstsrc_val;
6070         regarray[idx] = src;
6071
6072         /* Thread. */
6073         thread_ip_inc(p);
6074 }
6075
6076 static inline void
6077 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
6078 {
6079         struct thread *t = &p->threads[p->thread_id];
6080         struct instruction *ip = t->ip;
6081         uint64_t *regarray, idx, src;
6082
6083         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
6084
6085         /* Structs. */
6086         regarray = instr_regarray_regarray(p, ip);
6087         idx = instr_regarray_idx_hbo(p, t, ip);
6088         src = ip->regarray.dstsrc_val;
6089         regarray[idx] = src;
6090
6091         /* Thread. */
6092         thread_ip_inc(p);
6093 }
6094
6095 static inline void
6096 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
6097 {
6098         struct thread *t = &p->threads[p->thread_id];
6099         struct instruction *ip = t->ip;
6100         uint64_t *regarray, idx, src;
6101
6102         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
6103
6104         /* Structs. */
6105         regarray = instr_regarray_regarray(p, ip);
6106         idx = instr_regarray_idx_imm(p, ip);
6107         src = instr_regarray_src_nbo(t, ip);
6108         regarray[idx] = src;
6109
6110         /* Thread. */
6111         thread_ip_inc(p);
6112 }
6113
6114 static inline void
6115 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
6116 {
6117         struct thread *t = &p->threads[p->thread_id];
6118         struct instruction *ip = t->ip;
6119         uint64_t *regarray, idx, src;
6120
6121         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
6122
6123         /* Structs. */
6124         regarray = instr_regarray_regarray(p, ip);
6125         idx = instr_regarray_idx_imm(p, ip);
6126         src = instr_regarray_src_hbo(t, ip);
6127         regarray[idx] = src;
6128
6129         /* Thread. */
6130         thread_ip_inc(p);
6131 }
6132
6133 static inline void
6134 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
6135 {
6136         struct thread *t = &p->threads[p->thread_id];
6137         struct instruction *ip = t->ip;
6138         uint64_t *regarray, idx, src;
6139
6140         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
6141
6142         /* Structs. */
6143         regarray = instr_regarray_regarray(p, ip);
6144         idx = instr_regarray_idx_imm(p, ip);
6145         src = ip->regarray.dstsrc_val;
6146         regarray[idx] = src;
6147
6148         /* Thread. */
6149         thread_ip_inc(p);
6150 }
6151
6152 static inline void
6153 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
6154 {
6155         struct thread *t = &p->threads[p->thread_id];
6156         struct instruction *ip = t->ip;
6157         uint64_t *regarray, idx, src;
6158
6159         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
6160
6161         /* Structs. */
6162         regarray = instr_regarray_regarray(p, ip);
6163         idx = instr_regarray_idx_nbo(p, t, ip);
6164         src = instr_regarray_src_nbo(t, ip);
6165         regarray[idx] += src;
6166
6167         /* Thread. */
6168         thread_ip_inc(p);
6169 }
6170
6171 static inline void
6172 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
6173 {
6174         struct thread *t = &p->threads[p->thread_id];
6175         struct instruction *ip = t->ip;
6176         uint64_t *regarray, idx, src;
6177
6178         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
6179
6180         /* Structs. */
6181         regarray = instr_regarray_regarray(p, ip);
6182         idx = instr_regarray_idx_nbo(p, t, ip);
6183         src = instr_regarray_src_hbo(t, ip);
6184         regarray[idx] += src;
6185
6186         /* Thread. */
6187         thread_ip_inc(p);
6188 }
6189
6190 static inline void
6191 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
6192 {
6193         struct thread *t = &p->threads[p->thread_id];
6194         struct instruction *ip = t->ip;
6195         uint64_t *regarray, idx, src;
6196
6197         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
6198
6199         /* Structs. */
6200         regarray = instr_regarray_regarray(p, ip);
6201         idx = instr_regarray_idx_hbo(p, t, ip);
6202         src = instr_regarray_src_nbo(t, ip);
6203         regarray[idx] += src;
6204
6205         /* Thread. */
6206         thread_ip_inc(p);
6207 }
6208
6209 static inline void
6210 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
6211 {
6212         struct thread *t = &p->threads[p->thread_id];
6213         struct instruction *ip = t->ip;
6214         uint64_t *regarray, idx, src;
6215
6216         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
6217
6218         /* Structs. */
6219         regarray = instr_regarray_regarray(p, ip);
6220         idx = instr_regarray_idx_hbo(p, t, ip);
6221         src = instr_regarray_src_hbo(t, ip);
6222         regarray[idx] += src;
6223
6224         /* Thread. */
6225         thread_ip_inc(p);
6226 }
6227
6228 static inline void
6229 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
6230 {
6231         struct thread *t = &p->threads[p->thread_id];
6232         struct instruction *ip = t->ip;
6233         uint64_t *regarray, idx, src;
6234
6235         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
6236
6237         /* Structs. */
6238         regarray = instr_regarray_regarray(p, ip);
6239         idx = instr_regarray_idx_nbo(p, t, ip);
6240         src = ip->regarray.dstsrc_val;
6241         regarray[idx] += src;
6242
6243         /* Thread. */
6244         thread_ip_inc(p);
6245 }
6246
6247 static inline void
6248 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
6249 {
6250         struct thread *t = &p->threads[p->thread_id];
6251         struct instruction *ip = t->ip;
6252         uint64_t *regarray, idx, src;
6253
6254         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
6255
6256         /* Structs. */
6257         regarray = instr_regarray_regarray(p, ip);
6258         idx = instr_regarray_idx_hbo(p, t, ip);
6259         src = ip->regarray.dstsrc_val;
6260         regarray[idx] += src;
6261
6262         /* Thread. */
6263         thread_ip_inc(p);
6264 }
6265
6266 static inline void
6267 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
6268 {
6269         struct thread *t = &p->threads[p->thread_id];
6270         struct instruction *ip = t->ip;
6271         uint64_t *regarray, idx, src;
6272
6273         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
6274
6275         /* Structs. */
6276         regarray = instr_regarray_regarray(p, ip);
6277         idx = instr_regarray_idx_imm(p, ip);
6278         src = instr_regarray_src_nbo(t, ip);
6279         regarray[idx] += src;
6280
6281         /* Thread. */
6282         thread_ip_inc(p);
6283 }
6284
6285 static inline void
6286 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
6287 {
6288         struct thread *t = &p->threads[p->thread_id];
6289         struct instruction *ip = t->ip;
6290         uint64_t *regarray, idx, src;
6291
6292         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
6293
6294         /* Structs. */
6295         regarray = instr_regarray_regarray(p, ip);
6296         idx = instr_regarray_idx_imm(p, ip);
6297         src = instr_regarray_src_hbo(t, ip);
6298         regarray[idx] += src;
6299
6300         /* Thread. */
6301         thread_ip_inc(p);
6302 }
6303
6304 static inline void
6305 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
6306 {
6307         struct thread *t = &p->threads[p->thread_id];
6308         struct instruction *ip = t->ip;
6309         uint64_t *regarray, idx, src;
6310
6311         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
6312
6313         /* Structs. */
6314         regarray = instr_regarray_regarray(p, ip);
6315         idx = instr_regarray_idx_imm(p, ip);
6316         src = ip->regarray.dstsrc_val;
6317         regarray[idx] += src;
6318
6319         /* Thread. */
6320         thread_ip_inc(p);
6321 }
6322
6323 /*
6324  * metarray.
6325  */
6326 static struct metarray *
6327 metarray_find(struct rte_swx_pipeline *p, const char *name);
6328
6329 static int
6330 instr_metprefetch_translate(struct rte_swx_pipeline *p,
6331                             struct action *action,
6332                             char **tokens,
6333                             int n_tokens,
6334                             struct instruction *instr,
6335                             struct instruction_data *data __rte_unused)
6336 {
6337         char *metarray = tokens[1], *idx = tokens[2];
6338         struct metarray *m;
6339         struct field *fidx;
6340         uint32_t idx_struct_id, idx_val;
6341
6342         CHECK(n_tokens == 3, EINVAL);
6343
6344         m = metarray_find(p, metarray);
6345         CHECK(m, EINVAL);
6346
6347         /* METPREFETCH_H, METPREFETCH_M. */
6348         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6349         if (fidx) {
6350                 CHECK(!fidx->var_size, EINVAL);
6351
6352                 instr->type = INSTR_METPREFETCH_M;
6353                 if (idx[0] == 'h')
6354                         instr->type = INSTR_METPREFETCH_H;
6355
6356                 instr->meter.metarray_id = m->id;
6357                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6358                 instr->meter.idx.n_bits = fidx->n_bits;
6359                 instr->meter.idx.offset = fidx->offset / 8;
6360                 return 0;
6361         }
6362
6363         /* METPREFETCH_I. */
6364         idx_val = strtoul(idx, &idx, 0);
6365         CHECK(!idx[0], EINVAL);
6366
6367         instr->type = INSTR_METPREFETCH_I;
6368         instr->meter.metarray_id = m->id;
6369         instr->meter.idx_val = idx_val;
6370         return 0;
6371 }
6372
6373 static int
6374 instr_meter_translate(struct rte_swx_pipeline *p,
6375                       struct action *action,
6376                       char **tokens,
6377                       int n_tokens,
6378                       struct instruction *instr,
6379                       struct instruction_data *data __rte_unused)
6380 {
6381         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
6382         char *color_in = tokens[4], *color_out = tokens[5];
6383         struct metarray *m;
6384         struct field *fidx, *flength, *fcin, *fcout;
6385         uint32_t idx_struct_id, length_struct_id;
6386         uint32_t color_in_struct_id, color_out_struct_id;
6387
6388         CHECK(n_tokens == 6, EINVAL);
6389
6390         m = metarray_find(p, metarray);
6391         CHECK(m, EINVAL);
6392
6393         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6394
6395         flength = struct_field_parse(p, action, length, &length_struct_id);
6396         CHECK(flength, EINVAL);
6397         CHECK(!flength->var_size, EINVAL);
6398
6399         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
6400
6401         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
6402         CHECK(fcout, EINVAL);
6403         CHECK(!fcout->var_size, EINVAL);
6404
6405         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
6406         if (fidx && fcin) {
6407                 CHECK(!fidx->var_size, EINVAL);
6408                 CHECK(!fcin->var_size, EINVAL);
6409
6410                 instr->type = INSTR_METER_MMM;
6411                 if (idx[0] == 'h' && length[0] == 'h')
6412                         instr->type = INSTR_METER_HHM;
6413                 if (idx[0] == 'h' && length[0] != 'h')
6414                         instr->type = INSTR_METER_HMM;
6415                 if (idx[0] != 'h' && length[0] == 'h')
6416                         instr->type = INSTR_METER_MHM;
6417
6418                 instr->meter.metarray_id = m->id;
6419
6420                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6421                 instr->meter.idx.n_bits = fidx->n_bits;
6422                 instr->meter.idx.offset = fidx->offset / 8;
6423
6424                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6425                 instr->meter.length.n_bits = flength->n_bits;
6426                 instr->meter.length.offset = flength->offset / 8;
6427
6428                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6429                 instr->meter.color_in.n_bits = fcin->n_bits;
6430                 instr->meter.color_in.offset = fcin->offset / 8;
6431
6432                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6433                 instr->meter.color_out.n_bits = fcout->n_bits;
6434                 instr->meter.color_out.offset = fcout->offset / 8;
6435
6436                 return 0;
6437         }
6438
6439         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
6440         if (fidx && !fcin) {
6441                 uint32_t color_in_val;
6442
6443                 CHECK(!fidx->var_size, EINVAL);
6444
6445                 color_in_val = strtoul(color_in, &color_in, 0);
6446                 CHECK(!color_in[0], EINVAL);
6447
6448                 instr->type = INSTR_METER_MMI;
6449                 if (idx[0] == 'h' && length[0] == 'h')
6450                         instr->type = INSTR_METER_HHI;
6451                 if (idx[0] == 'h' && length[0] != 'h')
6452                         instr->type = INSTR_METER_HMI;
6453                 if (idx[0] != 'h' && length[0] == 'h')
6454                         instr->type = INSTR_METER_MHI;
6455
6456                 instr->meter.metarray_id = m->id;
6457
6458                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6459                 instr->meter.idx.n_bits = fidx->n_bits;
6460                 instr->meter.idx.offset = fidx->offset / 8;
6461
6462                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6463                 instr->meter.length.n_bits = flength->n_bits;
6464                 instr->meter.length.offset = flength->offset / 8;
6465
6466                 instr->meter.color_in_val = color_in_val;
6467
6468                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6469                 instr->meter.color_out.n_bits = fcout->n_bits;
6470                 instr->meter.color_out.offset = fcout->offset / 8;
6471
6472                 return 0;
6473         }
6474
6475         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
6476         if (!fidx && fcin) {
6477                 uint32_t idx_val;
6478
6479                 idx_val = strtoul(idx, &idx, 0);
6480                 CHECK(!idx[0], EINVAL);
6481
6482                 CHECK(!fcin->var_size, EINVAL);
6483
6484                 instr->type = INSTR_METER_IMM;
6485                 if (length[0] == 'h')
6486                         instr->type = INSTR_METER_IHM;
6487
6488                 instr->meter.metarray_id = m->id;
6489
6490                 instr->meter.idx_val = idx_val;
6491
6492                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6493                 instr->meter.length.n_bits = flength->n_bits;
6494                 instr->meter.length.offset = flength->offset / 8;
6495
6496                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6497                 instr->meter.color_in.n_bits = fcin->n_bits;
6498                 instr->meter.color_in.offset = fcin->offset / 8;
6499
6500                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6501                 instr->meter.color_out.n_bits = fcout->n_bits;
6502                 instr->meter.color_out.offset = fcout->offset / 8;
6503
6504                 return 0;
6505         }
6506
6507         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
6508         if (!fidx && !fcin) {
6509                 uint32_t idx_val, color_in_val;
6510
6511                 idx_val = strtoul(idx, &idx, 0);
6512                 CHECK(!idx[0], EINVAL);
6513
6514                 color_in_val = strtoul(color_in, &color_in, 0);
6515                 CHECK(!color_in[0], EINVAL);
6516
6517                 instr->type = INSTR_METER_IMI;
6518                 if (length[0] == 'h')
6519                         instr->type = INSTR_METER_IHI;
6520
6521                 instr->meter.metarray_id = m->id;
6522
6523                 instr->meter.idx_val = idx_val;
6524
6525                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6526                 instr->meter.length.n_bits = flength->n_bits;
6527                 instr->meter.length.offset = flength->offset / 8;
6528
6529                 instr->meter.color_in_val = color_in_val;
6530
6531                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6532                 instr->meter.color_out.n_bits = fcout->n_bits;
6533                 instr->meter.color_out.offset = fcout->offset / 8;
6534
6535                 return 0;
6536         }
6537
6538         CHECK(0, EINVAL);
6539 }
6540
6541 static inline struct meter *
6542 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6543 {
6544         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6545
6546         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6547         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6548         uint64_t idx64 = *idx64_ptr;
6549         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
6550         uint64_t idx = idx64 & idx64_mask & r->size_mask;
6551
6552         return &r->metarray[idx];
6553 }
6554
6555 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6556
6557 static inline struct meter *
6558 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6559 {
6560         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6561
6562         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6563         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6564         uint64_t idx64 = *idx64_ptr;
6565         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
6566
6567         return &r->metarray[idx];
6568 }
6569
6570 #else
6571
6572 #define instr_meter_idx_nbo instr_meter_idx_hbo
6573
6574 #endif
6575
6576 static inline struct meter *
6577 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
6578 {
6579         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6580
6581         uint64_t idx =  ip->meter.idx_val & r->size_mask;
6582
6583         return &r->metarray[idx];
6584 }
6585
6586 static inline uint32_t
6587 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
6588 {
6589         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6590         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6591         uint64_t src64 = *src64_ptr;
6592         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
6593         uint64_t src = src64 & src64_mask;
6594
6595         return (uint32_t)src;
6596 }
6597
6598 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6599
6600 static inline uint32_t
6601 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
6602 {
6603         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6604         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6605         uint64_t src64 = *src64_ptr;
6606         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
6607
6608         return (uint32_t)src;
6609 }
6610
6611 #else
6612
6613 #define instr_meter_length_nbo instr_meter_length_hbo
6614
6615 #endif
6616
6617 static inline enum rte_color
6618 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
6619 {
6620         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
6621         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
6622         uint64_t src64 = *src64_ptr;
6623         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
6624         uint64_t src = src64 & src64_mask;
6625
6626         return (enum rte_color)src;
6627 }
6628
6629 static inline void
6630 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
6631 {
6632         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
6633         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
6634         uint64_t dst64 = *dst64_ptr;
6635         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
6636
6637         uint64_t src = (uint64_t)color_out;
6638
6639         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6640 }
6641
6642 static inline void
6643 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
6644 {
6645         struct thread *t = &p->threads[p->thread_id];
6646         struct instruction *ip = t->ip;
6647         struct meter *m;
6648
6649         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
6650
6651         /* Structs. */
6652         m = instr_meter_idx_nbo(p, t, ip);
6653         rte_prefetch0(m);
6654
6655         /* Thread. */
6656         thread_ip_inc(p);
6657 }
6658
6659 static inline void
6660 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
6661 {
6662         struct thread *t = &p->threads[p->thread_id];
6663         struct instruction *ip = t->ip;
6664         struct meter *m;
6665
6666         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
6667
6668         /* Structs. */
6669         m = instr_meter_idx_hbo(p, t, ip);
6670         rte_prefetch0(m);
6671
6672         /* Thread. */
6673         thread_ip_inc(p);
6674 }
6675
6676 static inline void
6677 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
6678 {
6679         struct thread *t = &p->threads[p->thread_id];
6680         struct instruction *ip = t->ip;
6681         struct meter *m;
6682
6683         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
6684
6685         /* Structs. */
6686         m = instr_meter_idx_imm(p, ip);
6687         rte_prefetch0(m);
6688
6689         /* Thread. */
6690         thread_ip_inc(p);
6691 }
6692
6693 static inline void
6694 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
6695 {
6696         struct thread *t = &p->threads[p->thread_id];
6697         struct instruction *ip = t->ip;
6698         struct meter *m;
6699         uint64_t time, n_pkts, n_bytes;
6700         uint32_t length;
6701         enum rte_color color_in, color_out;
6702
6703         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
6704
6705         /* Structs. */
6706         m = instr_meter_idx_nbo(p, t, ip);
6707         rte_prefetch0(m->n_pkts);
6708         time = rte_get_tsc_cycles();
6709         length = instr_meter_length_nbo(t, ip);
6710         color_in = instr_meter_color_in_hbo(t, ip);
6711
6712         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6713                 &m->profile->profile,
6714                 time,
6715                 length,
6716                 color_in);
6717
6718         color_out &= m->color_mask;
6719
6720         n_pkts = m->n_pkts[color_out];
6721         n_bytes = m->n_bytes[color_out];
6722
6723         instr_meter_color_out_hbo_set(t, ip, color_out);
6724
6725         m->n_pkts[color_out] = n_pkts + 1;
6726         m->n_bytes[color_out] = n_bytes + length;
6727
6728         /* Thread. */
6729         thread_ip_inc(p);
6730 }
6731
6732 static inline void
6733 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
6734 {
6735         struct thread *t = &p->threads[p->thread_id];
6736         struct instruction *ip = t->ip;
6737         struct meter *m;
6738         uint64_t time, n_pkts, n_bytes;
6739         uint32_t length;
6740         enum rte_color color_in, color_out;
6741
6742         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
6743
6744         /* Structs. */
6745         m = instr_meter_idx_nbo(p, t, ip);
6746         rte_prefetch0(m->n_pkts);
6747         time = rte_get_tsc_cycles();
6748         length = instr_meter_length_nbo(t, ip);
6749         color_in = (enum rte_color)ip->meter.color_in_val;
6750
6751         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6752                 &m->profile->profile,
6753                 time,
6754                 length,
6755                 color_in);
6756
6757         color_out &= m->color_mask;
6758
6759         n_pkts = m->n_pkts[color_out];
6760         n_bytes = m->n_bytes[color_out];
6761
6762         instr_meter_color_out_hbo_set(t, ip, color_out);
6763
6764         m->n_pkts[color_out] = n_pkts + 1;
6765         m->n_bytes[color_out] = n_bytes + length;
6766
6767         /* Thread. */
6768         thread_ip_inc(p);
6769 }
6770
6771 static inline void
6772 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
6773 {
6774         struct thread *t = &p->threads[p->thread_id];
6775         struct instruction *ip = t->ip;
6776         struct meter *m;
6777         uint64_t time, n_pkts, n_bytes;
6778         uint32_t length;
6779         enum rte_color color_in, color_out;
6780
6781         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
6782
6783         /* Structs. */
6784         m = instr_meter_idx_nbo(p, t, ip);
6785         rte_prefetch0(m->n_pkts);
6786         time = rte_get_tsc_cycles();
6787         length = instr_meter_length_hbo(t, ip);
6788         color_in = instr_meter_color_in_hbo(t, ip);
6789
6790         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6791                 &m->profile->profile,
6792                 time,
6793                 length,
6794                 color_in);
6795
6796         color_out &= m->color_mask;
6797
6798         n_pkts = m->n_pkts[color_out];
6799         n_bytes = m->n_bytes[color_out];
6800
6801         instr_meter_color_out_hbo_set(t, ip, color_out);
6802
6803         m->n_pkts[color_out] = n_pkts + 1;
6804         m->n_bytes[color_out] = n_bytes + length;
6805
6806         /* Thread. */
6807         thread_ip_inc(p);
6808 }
6809 static inline void
6810 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
6811 {
6812         struct thread *t = &p->threads[p->thread_id];
6813         struct instruction *ip = t->ip;
6814         struct meter *m;
6815         uint64_t time, n_pkts, n_bytes;
6816         uint32_t length;
6817         enum rte_color color_in, color_out;
6818
6819         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
6820
6821         /* Structs. */
6822         m = instr_meter_idx_nbo(p, t, ip);
6823         rte_prefetch0(m->n_pkts);
6824         time = rte_get_tsc_cycles();
6825         length = instr_meter_length_hbo(t, ip);
6826         color_in = (enum rte_color)ip->meter.color_in_val;
6827
6828         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6829                 &m->profile->profile,
6830                 time,
6831                 length,
6832                 color_in);
6833
6834         color_out &= m->color_mask;
6835
6836         n_pkts = m->n_pkts[color_out];
6837         n_bytes = m->n_bytes[color_out];
6838
6839         instr_meter_color_out_hbo_set(t, ip, color_out);
6840
6841         m->n_pkts[color_out] = n_pkts + 1;
6842         m->n_bytes[color_out] = n_bytes + length;
6843
6844         /* Thread. */
6845         thread_ip_inc(p);
6846 }
6847
6848 static inline void
6849 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
6850 {
6851         struct thread *t = &p->threads[p->thread_id];
6852         struct instruction *ip = t->ip;
6853         struct meter *m;
6854         uint64_t time, n_pkts, n_bytes;
6855         uint32_t length;
6856         enum rte_color color_in, color_out;
6857
6858         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
6859
6860         /* Structs. */
6861         m = instr_meter_idx_hbo(p, t, ip);
6862         rte_prefetch0(m->n_pkts);
6863         time = rte_get_tsc_cycles();
6864         length = instr_meter_length_nbo(t, ip);
6865         color_in = instr_meter_color_in_hbo(t, ip);
6866
6867         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6868                 &m->profile->profile,
6869                 time,
6870                 length,
6871                 color_in);
6872
6873         color_out &= m->color_mask;
6874
6875         n_pkts = m->n_pkts[color_out];
6876         n_bytes = m->n_bytes[color_out];
6877
6878         instr_meter_color_out_hbo_set(t, ip, color_out);
6879
6880         m->n_pkts[color_out] = n_pkts + 1;
6881         m->n_bytes[color_out] = n_bytes + length;
6882
6883         /* Thread. */
6884         thread_ip_inc(p);
6885 }
6886
6887 static inline void
6888 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
6889 {
6890         struct thread *t = &p->threads[p->thread_id];
6891         struct instruction *ip = t->ip;
6892         struct meter *m;
6893         uint64_t time, n_pkts, n_bytes;
6894         uint32_t length;
6895         enum rte_color color_in, color_out;
6896
6897         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
6898
6899         /* Structs. */
6900         m = instr_meter_idx_hbo(p, t, ip);
6901         rte_prefetch0(m->n_pkts);
6902         time = rte_get_tsc_cycles();
6903         length = instr_meter_length_nbo(t, ip);
6904         color_in = (enum rte_color)ip->meter.color_in_val;
6905
6906         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6907                 &m->profile->profile,
6908                 time,
6909                 length,
6910                 color_in);
6911
6912         color_out &= m->color_mask;
6913
6914         n_pkts = m->n_pkts[color_out];
6915         n_bytes = m->n_bytes[color_out];
6916
6917         instr_meter_color_out_hbo_set(t, ip, color_out);
6918
6919         m->n_pkts[color_out] = n_pkts + 1;
6920         m->n_bytes[color_out] = n_bytes + length;
6921
6922         /* Thread. */
6923         thread_ip_inc(p);
6924 }
6925
6926 static inline void
6927 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
6928 {
6929         struct thread *t = &p->threads[p->thread_id];
6930         struct instruction *ip = t->ip;
6931         struct meter *m;
6932         uint64_t time, n_pkts, n_bytes;
6933         uint32_t length;
6934         enum rte_color color_in, color_out;
6935
6936         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
6937
6938         /* Structs. */
6939         m = instr_meter_idx_hbo(p, t, ip);
6940         rte_prefetch0(m->n_pkts);
6941         time = rte_get_tsc_cycles();
6942         length = instr_meter_length_hbo(t, ip);
6943         color_in = instr_meter_color_in_hbo(t, ip);
6944
6945         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6946                 &m->profile->profile,
6947                 time,
6948                 length,
6949                 color_in);
6950
6951         color_out &= m->color_mask;
6952
6953         n_pkts = m->n_pkts[color_out];
6954         n_bytes = m->n_bytes[color_out];
6955
6956         instr_meter_color_out_hbo_set(t, ip, color_out);
6957
6958         m->n_pkts[color_out] = n_pkts + 1;
6959         m->n_bytes[color_out] = n_bytes + length;
6960
6961         /* Thread. */
6962         thread_ip_inc(p);
6963 }
6964
6965 static inline void
6966 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
6967 {
6968         struct thread *t = &p->threads[p->thread_id];
6969         struct instruction *ip = t->ip;
6970         struct meter *m;
6971         uint64_t time, n_pkts, n_bytes;
6972         uint32_t length;
6973         enum rte_color color_in, color_out;
6974
6975         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
6976
6977         /* Structs. */
6978         m = instr_meter_idx_hbo(p, t, ip);
6979         rte_prefetch0(m->n_pkts);
6980         time = rte_get_tsc_cycles();
6981         length = instr_meter_length_hbo(t, ip);
6982         color_in = (enum rte_color)ip->meter.color_in_val;
6983
6984         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6985                 &m->profile->profile,
6986                 time,
6987                 length,
6988                 color_in);
6989
6990         color_out &= m->color_mask;
6991
6992         n_pkts = m->n_pkts[color_out];
6993         n_bytes = m->n_bytes[color_out];
6994
6995         instr_meter_color_out_hbo_set(t, ip, color_out);
6996
6997         m->n_pkts[color_out] = n_pkts + 1;
6998         m->n_bytes[color_out] = n_bytes + length;
6999
7000         /* Thread. */
7001         thread_ip_inc(p);
7002 }
7003
7004 static inline void
7005 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
7006 {
7007         struct thread *t = &p->threads[p->thread_id];
7008         struct instruction *ip = t->ip;
7009         struct meter *m;
7010         uint64_t time, n_pkts, n_bytes;
7011         uint32_t length;
7012         enum rte_color color_in, color_out;
7013
7014         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
7015
7016         /* Structs. */
7017         m = instr_meter_idx_imm(p, ip);
7018         rte_prefetch0(m->n_pkts);
7019         time = rte_get_tsc_cycles();
7020         length = instr_meter_length_nbo(t, ip);
7021         color_in = instr_meter_color_in_hbo(t, ip);
7022
7023         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7024                 &m->profile->profile,
7025                 time,
7026                 length,
7027                 color_in);
7028
7029         color_out &= m->color_mask;
7030
7031         n_pkts = m->n_pkts[color_out];
7032         n_bytes = m->n_bytes[color_out];
7033
7034         instr_meter_color_out_hbo_set(t, ip, color_out);
7035
7036         m->n_pkts[color_out] = n_pkts + 1;
7037         m->n_bytes[color_out] = n_bytes + length;
7038
7039         /* Thread. */
7040         thread_ip_inc(p);
7041 }
7042
7043 static inline void
7044 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
7045 {
7046         struct thread *t = &p->threads[p->thread_id];
7047         struct instruction *ip = t->ip;
7048         struct meter *m;
7049         uint64_t time, n_pkts, n_bytes;
7050         uint32_t length;
7051         enum rte_color color_in, color_out;
7052
7053         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
7054
7055         /* Structs. */
7056         m = instr_meter_idx_imm(p, ip);
7057         rte_prefetch0(m->n_pkts);
7058         time = rte_get_tsc_cycles();
7059         length = instr_meter_length_nbo(t, ip);
7060         color_in = (enum rte_color)ip->meter.color_in_val;
7061
7062         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7063                 &m->profile->profile,
7064                 time,
7065                 length,
7066                 color_in);
7067
7068         color_out &= m->color_mask;
7069
7070         n_pkts = m->n_pkts[color_out];
7071         n_bytes = m->n_bytes[color_out];
7072
7073         instr_meter_color_out_hbo_set(t, ip, color_out);
7074
7075         m->n_pkts[color_out] = n_pkts + 1;
7076         m->n_bytes[color_out] = n_bytes + length;
7077
7078         /* Thread. */
7079         thread_ip_inc(p);
7080 }
7081
7082 static inline void
7083 instr_meter_imm_exec(struct rte_swx_pipeline *p)
7084 {
7085         struct thread *t = &p->threads[p->thread_id];
7086         struct instruction *ip = t->ip;
7087         struct meter *m;
7088         uint64_t time, n_pkts, n_bytes;
7089         uint32_t length;
7090         enum rte_color color_in, color_out;
7091
7092         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
7093
7094         /* Structs. */
7095         m = instr_meter_idx_imm(p, ip);
7096         rte_prefetch0(m->n_pkts);
7097         time = rte_get_tsc_cycles();
7098         length = instr_meter_length_hbo(t, ip);
7099         color_in = instr_meter_color_in_hbo(t, ip);
7100
7101         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7102                 &m->profile->profile,
7103                 time,
7104                 length,
7105                 color_in);
7106
7107         color_out &= m->color_mask;
7108
7109         n_pkts = m->n_pkts[color_out];
7110         n_bytes = m->n_bytes[color_out];
7111
7112         instr_meter_color_out_hbo_set(t, ip, color_out);
7113
7114         m->n_pkts[color_out] = n_pkts + 1;
7115         m->n_bytes[color_out] = n_bytes + length;
7116
7117         /* Thread. */
7118         thread_ip_inc(p);
7119 }
7120 static inline void
7121 instr_meter_imi_exec(struct rte_swx_pipeline *p)
7122 {
7123         struct thread *t = &p->threads[p->thread_id];
7124         struct instruction *ip = t->ip;
7125         struct meter *m;
7126         uint64_t time, n_pkts, n_bytes;
7127         uint32_t length;
7128         enum rte_color color_in, color_out;
7129
7130         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
7131
7132         /* Structs. */
7133         m = instr_meter_idx_imm(p, ip);
7134         rte_prefetch0(m->n_pkts);
7135         time = rte_get_tsc_cycles();
7136         length = instr_meter_length_hbo(t, ip);
7137         color_in = (enum rte_color)ip->meter.color_in_val;
7138
7139         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7140                 &m->profile->profile,
7141                 time,
7142                 length,
7143                 color_in);
7144
7145         color_out &= m->color_mask;
7146
7147         n_pkts = m->n_pkts[color_out];
7148         n_bytes = m->n_bytes[color_out];
7149
7150         instr_meter_color_out_hbo_set(t, ip, color_out);
7151
7152         m->n_pkts[color_out] = n_pkts + 1;
7153         m->n_bytes[color_out] = n_bytes + length;
7154
7155         /* Thread. */
7156         thread_ip_inc(p);
7157 }
7158
7159 /*
7160  * jmp.
7161  */
7162 static struct action *
7163 action_find(struct rte_swx_pipeline *p, const char *name);
7164
7165 static int
7166 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
7167                     struct action *action __rte_unused,
7168                     char **tokens,
7169                     int n_tokens,
7170                     struct instruction *instr,
7171                     struct instruction_data *data)
7172 {
7173         CHECK(n_tokens == 2, EINVAL);
7174
7175         strcpy(data->jmp_label, tokens[1]);
7176
7177         instr->type = INSTR_JMP;
7178         instr->jmp.ip = NULL; /* Resolved later. */
7179         return 0;
7180 }
7181
7182 static int
7183 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
7184                           struct action *action __rte_unused,
7185                           char **tokens,
7186                           int n_tokens,
7187                           struct instruction *instr,
7188                           struct instruction_data *data)
7189 {
7190         struct header *h;
7191
7192         CHECK(n_tokens == 3, EINVAL);
7193
7194         strcpy(data->jmp_label, tokens[1]);
7195
7196         h = header_parse(p, tokens[2]);
7197         CHECK(h, EINVAL);
7198
7199         instr->type = INSTR_JMP_VALID;
7200         instr->jmp.ip = NULL; /* Resolved later. */
7201         instr->jmp.header_id = h->id;
7202         return 0;
7203 }
7204
7205 static int
7206 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
7207                             struct action *action __rte_unused,
7208                             char **tokens,
7209                             int n_tokens,
7210                             struct instruction *instr,
7211                             struct instruction_data *data)
7212 {
7213         struct header *h;
7214
7215         CHECK(n_tokens == 3, EINVAL);
7216
7217         strcpy(data->jmp_label, tokens[1]);
7218
7219         h = header_parse(p, tokens[2]);
7220         CHECK(h, EINVAL);
7221
7222         instr->type = INSTR_JMP_INVALID;
7223         instr->jmp.ip = NULL; /* Resolved later. */
7224         instr->jmp.header_id = h->id;
7225         return 0;
7226 }
7227
7228 static int
7229 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
7230                         struct action *action,
7231                         char **tokens,
7232                         int n_tokens,
7233                         struct instruction *instr,
7234                         struct instruction_data *data)
7235 {
7236         CHECK(!action, EINVAL);
7237         CHECK(n_tokens == 2, EINVAL);
7238
7239         strcpy(data->jmp_label, tokens[1]);
7240
7241         instr->type = INSTR_JMP_HIT;
7242         instr->jmp.ip = NULL; /* Resolved later. */
7243         return 0;
7244 }
7245
7246 static int
7247 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
7248                          struct action *action,
7249                          char **tokens,
7250                          int n_tokens,
7251                          struct instruction *instr,
7252                          struct instruction_data *data)
7253 {
7254         CHECK(!action, EINVAL);
7255         CHECK(n_tokens == 2, EINVAL);
7256
7257         strcpy(data->jmp_label, tokens[1]);
7258
7259         instr->type = INSTR_JMP_MISS;
7260         instr->jmp.ip = NULL; /* Resolved later. */
7261         return 0;
7262 }
7263
7264 static int
7265 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
7266                                struct action *action,
7267                                char **tokens,
7268                                int n_tokens,
7269                                struct instruction *instr,
7270                                struct instruction_data *data)
7271 {
7272         struct action *a;
7273
7274         CHECK(!action, EINVAL);
7275         CHECK(n_tokens == 3, EINVAL);
7276
7277         strcpy(data->jmp_label, tokens[1]);
7278
7279         a = action_find(p, tokens[2]);
7280         CHECK(a, EINVAL);
7281
7282         instr->type = INSTR_JMP_ACTION_HIT;
7283         instr->jmp.ip = NULL; /* Resolved later. */
7284         instr->jmp.action_id = a->id;
7285         return 0;
7286 }
7287
7288 static int
7289 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
7290                                 struct action *action,
7291                                 char **tokens,
7292                                 int n_tokens,
7293                                 struct instruction *instr,
7294                                 struct instruction_data *data)
7295 {
7296         struct action *a;
7297
7298         CHECK(!action, EINVAL);
7299         CHECK(n_tokens == 3, EINVAL);
7300
7301         strcpy(data->jmp_label, tokens[1]);
7302
7303         a = action_find(p, tokens[2]);
7304         CHECK(a, EINVAL);
7305
7306         instr->type = INSTR_JMP_ACTION_MISS;
7307         instr->jmp.ip = NULL; /* Resolved later. */
7308         instr->jmp.action_id = a->id;
7309         return 0;
7310 }
7311
7312 static int
7313 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
7314                        struct action *action,
7315                        char **tokens,
7316                        int n_tokens,
7317                        struct instruction *instr,
7318                        struct instruction_data *data)
7319 {
7320         char *a = tokens[2], *b = tokens[3];
7321         struct field *fa, *fb;
7322         uint64_t b_val;
7323         uint32_t a_struct_id, b_struct_id;
7324
7325         CHECK(n_tokens == 4, EINVAL);
7326
7327         strcpy(data->jmp_label, tokens[1]);
7328
7329         fa = struct_field_parse(p, action, a, &a_struct_id);
7330         CHECK(fa, EINVAL);
7331         CHECK(!fa->var_size, EINVAL);
7332
7333         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
7334         fb = struct_field_parse(p, action, b, &b_struct_id);
7335         if (fb) {
7336                 CHECK(!fb->var_size, EINVAL);
7337
7338                 instr->type = INSTR_JMP_EQ;
7339                 if (a[0] != 'h' && b[0] == 'h')
7340                         instr->type = INSTR_JMP_EQ_MH;
7341                 if (a[0] == 'h' && b[0] != 'h')
7342                         instr->type = INSTR_JMP_EQ_HM;
7343                 if (a[0] == 'h' && b[0] == 'h')
7344                         instr->type = INSTR_JMP_EQ_HH;
7345                 instr->jmp.ip = NULL; /* Resolved later. */
7346
7347                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7348                 instr->jmp.a.n_bits = fa->n_bits;
7349                 instr->jmp.a.offset = fa->offset / 8;
7350                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7351                 instr->jmp.b.n_bits = fb->n_bits;
7352                 instr->jmp.b.offset = fb->offset / 8;
7353                 return 0;
7354         }
7355
7356         /* JMP_EQ_I. */
7357         b_val = strtoull(b, &b, 0);
7358         CHECK(!b[0], EINVAL);
7359
7360         if (a[0] == 'h')
7361                 b_val = hton64(b_val) >> (64 - fa->n_bits);
7362
7363         instr->type = INSTR_JMP_EQ_I;
7364         instr->jmp.ip = NULL; /* Resolved later. */
7365         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7366         instr->jmp.a.n_bits = fa->n_bits;
7367         instr->jmp.a.offset = fa->offset / 8;
7368         instr->jmp.b_val = b_val;
7369         return 0;
7370 }
7371
7372 static int
7373 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
7374                         struct action *action,
7375                         char **tokens,
7376                         int n_tokens,
7377                         struct instruction *instr,
7378                         struct instruction_data *data)
7379 {
7380         char *a = tokens[2], *b = tokens[3];
7381         struct field *fa, *fb;
7382         uint64_t b_val;
7383         uint32_t a_struct_id, b_struct_id;
7384
7385         CHECK(n_tokens == 4, EINVAL);
7386
7387         strcpy(data->jmp_label, tokens[1]);
7388
7389         fa = struct_field_parse(p, action, a, &a_struct_id);
7390         CHECK(fa, EINVAL);
7391         CHECK(!fa->var_size, EINVAL);
7392
7393         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
7394         fb = struct_field_parse(p, action, b, &b_struct_id);
7395         if (fb) {
7396                 CHECK(!fb->var_size, EINVAL);
7397
7398                 instr->type = INSTR_JMP_NEQ;
7399                 if (a[0] != 'h' && b[0] == 'h')
7400                         instr->type = INSTR_JMP_NEQ_MH;
7401                 if (a[0] == 'h' && b[0] != 'h')
7402                         instr->type = INSTR_JMP_NEQ_HM;
7403                 if (a[0] == 'h' && b[0] == 'h')
7404                         instr->type = INSTR_JMP_NEQ_HH;
7405                 instr->jmp.ip = NULL; /* Resolved later. */
7406
7407                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7408                 instr->jmp.a.n_bits = fa->n_bits;
7409                 instr->jmp.a.offset = fa->offset / 8;
7410                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7411                 instr->jmp.b.n_bits = fb->n_bits;
7412                 instr->jmp.b.offset = fb->offset / 8;
7413                 return 0;
7414         }
7415
7416         /* JMP_NEQ_I. */
7417         b_val = strtoull(b, &b, 0);
7418         CHECK(!b[0], EINVAL);
7419
7420         if (a[0] == 'h')
7421                 b_val = hton64(b_val) >> (64 - fa->n_bits);
7422
7423         instr->type = INSTR_JMP_NEQ_I;
7424         instr->jmp.ip = NULL; /* Resolved later. */
7425         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7426         instr->jmp.a.n_bits = fa->n_bits;
7427         instr->jmp.a.offset = fa->offset / 8;
7428         instr->jmp.b_val = b_val;
7429         return 0;
7430 }
7431
7432 static int
7433 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
7434                        struct action *action,
7435                        char **tokens,
7436                        int n_tokens,
7437                        struct instruction *instr,
7438                        struct instruction_data *data)
7439 {
7440         char *a = tokens[2], *b = tokens[3];
7441         struct field *fa, *fb;
7442         uint64_t b_val;
7443         uint32_t a_struct_id, b_struct_id;
7444
7445         CHECK(n_tokens == 4, EINVAL);
7446
7447         strcpy(data->jmp_label, tokens[1]);
7448
7449         fa = struct_field_parse(p, action, a, &a_struct_id);
7450         CHECK(fa, EINVAL);
7451         CHECK(!fa->var_size, EINVAL);
7452
7453         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
7454         fb = struct_field_parse(p, action, b, &b_struct_id);
7455         if (fb) {
7456                 CHECK(!fb->var_size, EINVAL);
7457
7458                 instr->type = INSTR_JMP_LT;
7459                 if (a[0] == 'h' && b[0] != 'h')
7460                         instr->type = INSTR_JMP_LT_HM;
7461                 if (a[0] != 'h' && b[0] == 'h')
7462                         instr->type = INSTR_JMP_LT_MH;
7463                 if (a[0] == 'h' && b[0] == 'h')
7464                         instr->type = INSTR_JMP_LT_HH;
7465                 instr->jmp.ip = NULL; /* Resolved later. */
7466
7467                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7468                 instr->jmp.a.n_bits = fa->n_bits;
7469                 instr->jmp.a.offset = fa->offset / 8;
7470                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7471                 instr->jmp.b.n_bits = fb->n_bits;
7472                 instr->jmp.b.offset = fb->offset / 8;
7473                 return 0;
7474         }
7475
7476         /* JMP_LT_MI, JMP_LT_HI. */
7477         b_val = strtoull(b, &b, 0);
7478         CHECK(!b[0], EINVAL);
7479
7480         instr->type = INSTR_JMP_LT_MI;
7481         if (a[0] == 'h')
7482                 instr->type = INSTR_JMP_LT_HI;
7483         instr->jmp.ip = NULL; /* Resolved later. */
7484
7485         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7486         instr->jmp.a.n_bits = fa->n_bits;
7487         instr->jmp.a.offset = fa->offset / 8;
7488         instr->jmp.b_val = b_val;
7489         return 0;
7490 }
7491
7492 static int
7493 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
7494                        struct action *action,
7495                        char **tokens,
7496                        int n_tokens,
7497                        struct instruction *instr,
7498                        struct instruction_data *data)
7499 {
7500         char *a = tokens[2], *b = tokens[3];
7501         struct field *fa, *fb;
7502         uint64_t b_val;
7503         uint32_t a_struct_id, b_struct_id;
7504
7505         CHECK(n_tokens == 4, EINVAL);
7506
7507         strcpy(data->jmp_label, tokens[1]);
7508
7509         fa = struct_field_parse(p, action, a, &a_struct_id);
7510         CHECK(fa, EINVAL);
7511         CHECK(!fa->var_size, EINVAL);
7512
7513         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
7514         fb = struct_field_parse(p, action, b, &b_struct_id);
7515         if (fb) {
7516                 CHECK(!fb->var_size, EINVAL);
7517
7518                 instr->type = INSTR_JMP_GT;
7519                 if (a[0] == 'h' && b[0] != 'h')
7520                         instr->type = INSTR_JMP_GT_HM;
7521                 if (a[0] != 'h' && b[0] == 'h')
7522                         instr->type = INSTR_JMP_GT_MH;
7523                 if (a[0] == 'h' && b[0] == 'h')
7524                         instr->type = INSTR_JMP_GT_HH;
7525                 instr->jmp.ip = NULL; /* Resolved later. */
7526
7527                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7528                 instr->jmp.a.n_bits = fa->n_bits;
7529                 instr->jmp.a.offset = fa->offset / 8;
7530                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7531                 instr->jmp.b.n_bits = fb->n_bits;
7532                 instr->jmp.b.offset = fb->offset / 8;
7533                 return 0;
7534         }
7535
7536         /* JMP_GT_MI, JMP_GT_HI. */
7537         b_val = strtoull(b, &b, 0);
7538         CHECK(!b[0], EINVAL);
7539
7540         instr->type = INSTR_JMP_GT_MI;
7541         if (a[0] == 'h')
7542                 instr->type = INSTR_JMP_GT_HI;
7543         instr->jmp.ip = NULL; /* Resolved later. */
7544
7545         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7546         instr->jmp.a.n_bits = fa->n_bits;
7547         instr->jmp.a.offset = fa->offset / 8;
7548         instr->jmp.b_val = b_val;
7549         return 0;
7550 }
7551
7552 static inline void
7553 instr_jmp_exec(struct rte_swx_pipeline *p)
7554 {
7555         struct thread *t = &p->threads[p->thread_id];
7556         struct instruction *ip = t->ip;
7557
7558         TRACE("[Thread %2u] jmp\n", p->thread_id);
7559
7560         thread_ip_set(t, ip->jmp.ip);
7561 }
7562
7563 static inline void
7564 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
7565 {
7566         struct thread *t = &p->threads[p->thread_id];
7567         struct instruction *ip = t->ip;
7568         uint32_t header_id = ip->jmp.header_id;
7569
7570         TRACE("[Thread %2u] jmpv\n", p->thread_id);
7571
7572         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
7573 }
7574
7575 static inline void
7576 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
7577 {
7578         struct thread *t = &p->threads[p->thread_id];
7579         struct instruction *ip = t->ip;
7580         uint32_t header_id = ip->jmp.header_id;
7581
7582         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
7583
7584         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
7585 }
7586
7587 static inline void
7588 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
7589 {
7590         struct thread *t = &p->threads[p->thread_id];
7591         struct instruction *ip = t->ip;
7592         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
7593
7594         TRACE("[Thread %2u] jmph\n", p->thread_id);
7595
7596         t->ip = ip_next[t->hit];
7597 }
7598
7599 static inline void
7600 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
7601 {
7602         struct thread *t = &p->threads[p->thread_id];
7603         struct instruction *ip = t->ip;
7604         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
7605
7606         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
7607
7608         t->ip = ip_next[t->hit];
7609 }
7610
7611 static inline void
7612 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
7613 {
7614         struct thread *t = &p->threads[p->thread_id];
7615         struct instruction *ip = t->ip;
7616
7617         TRACE("[Thread %2u] jmpa\n", p->thread_id);
7618
7619         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
7620 }
7621
7622 static inline void
7623 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
7624 {
7625         struct thread *t = &p->threads[p->thread_id];
7626         struct instruction *ip = t->ip;
7627
7628         TRACE("[Thread %2u] jmpna\n", p->thread_id);
7629
7630         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
7631 }
7632
7633 static inline void
7634 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
7635 {
7636         struct thread *t = &p->threads[p->thread_id];
7637         struct instruction *ip = t->ip;
7638
7639         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
7640
7641         JMP_CMP(t, ip, ==);
7642 }
7643
7644 static inline void
7645 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
7646 {
7647         struct thread *t = &p->threads[p->thread_id];
7648         struct instruction *ip = t->ip;
7649
7650         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
7651
7652         JMP_CMP_MH(t, ip, ==);
7653 }
7654
7655 static inline void
7656 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
7657 {
7658         struct thread *t = &p->threads[p->thread_id];
7659         struct instruction *ip = t->ip;
7660
7661         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
7662
7663         JMP_CMP_HM(t, ip, ==);
7664 }
7665
7666 static inline void
7667 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
7668 {
7669         struct thread *t = &p->threads[p->thread_id];
7670         struct instruction *ip = t->ip;
7671
7672         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
7673
7674         JMP_CMP_HH_FAST(t, ip, ==);
7675 }
7676
7677 static inline void
7678 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
7679 {
7680         struct thread *t = &p->threads[p->thread_id];
7681         struct instruction *ip = t->ip;
7682
7683         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
7684
7685         JMP_CMP_I(t, ip, ==);
7686 }
7687
7688 static inline void
7689 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
7690 {
7691         struct thread *t = &p->threads[p->thread_id];
7692         struct instruction *ip = t->ip;
7693
7694         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
7695
7696         JMP_CMP(t, ip, !=);
7697 }
7698
7699 static inline void
7700 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
7701 {
7702         struct thread *t = &p->threads[p->thread_id];
7703         struct instruction *ip = t->ip;
7704
7705         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
7706
7707         JMP_CMP_MH(t, ip, !=);
7708 }
7709
7710 static inline void
7711 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
7712 {
7713         struct thread *t = &p->threads[p->thread_id];
7714         struct instruction *ip = t->ip;
7715
7716         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
7717
7718         JMP_CMP_HM(t, ip, !=);
7719 }
7720
7721 static inline void
7722 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
7723 {
7724         struct thread *t = &p->threads[p->thread_id];
7725         struct instruction *ip = t->ip;
7726
7727         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
7728
7729         JMP_CMP_HH_FAST(t, ip, !=);
7730 }
7731
7732 static inline void
7733 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
7734 {
7735         struct thread *t = &p->threads[p->thread_id];
7736         struct instruction *ip = t->ip;
7737
7738         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
7739
7740         JMP_CMP_I(t, ip, !=);
7741 }
7742
7743 static inline void
7744 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
7745 {
7746         struct thread *t = &p->threads[p->thread_id];
7747         struct instruction *ip = t->ip;
7748
7749         TRACE("[Thread %2u] jmplt\n", p->thread_id);
7750
7751         JMP_CMP(t, ip, <);
7752 }
7753
7754 static inline void
7755 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
7756 {
7757         struct thread *t = &p->threads[p->thread_id];
7758         struct instruction *ip = t->ip;
7759
7760         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
7761
7762         JMP_CMP_MH(t, ip, <);
7763 }
7764
7765 static inline void
7766 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
7767 {
7768         struct thread *t = &p->threads[p->thread_id];
7769         struct instruction *ip = t->ip;
7770
7771         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
7772
7773         JMP_CMP_HM(t, ip, <);
7774 }
7775
7776 static inline void
7777 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
7778 {
7779         struct thread *t = &p->threads[p->thread_id];
7780         struct instruction *ip = t->ip;
7781
7782         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
7783
7784         JMP_CMP_HH(t, ip, <);
7785 }
7786
7787 static inline void
7788 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
7789 {
7790         struct thread *t = &p->threads[p->thread_id];
7791         struct instruction *ip = t->ip;
7792
7793         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
7794
7795         JMP_CMP_MI(t, ip, <);
7796 }
7797
7798 static inline void
7799 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
7800 {
7801         struct thread *t = &p->threads[p->thread_id];
7802         struct instruction *ip = t->ip;
7803
7804         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
7805
7806         JMP_CMP_HI(t, ip, <);
7807 }
7808
7809 static inline void
7810 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
7811 {
7812         struct thread *t = &p->threads[p->thread_id];
7813         struct instruction *ip = t->ip;
7814
7815         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
7816
7817         JMP_CMP(t, ip, >);
7818 }
7819
7820 static inline void
7821 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
7822 {
7823         struct thread *t = &p->threads[p->thread_id];
7824         struct instruction *ip = t->ip;
7825
7826         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
7827
7828         JMP_CMP_MH(t, ip, >);
7829 }
7830
7831 static inline void
7832 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
7833 {
7834         struct thread *t = &p->threads[p->thread_id];
7835         struct instruction *ip = t->ip;
7836
7837         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
7838
7839         JMP_CMP_HM(t, ip, >);
7840 }
7841
7842 static inline void
7843 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
7844 {
7845         struct thread *t = &p->threads[p->thread_id];
7846         struct instruction *ip = t->ip;
7847
7848         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
7849
7850         JMP_CMP_HH(t, ip, >);
7851 }
7852
7853 static inline void
7854 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
7855 {
7856         struct thread *t = &p->threads[p->thread_id];
7857         struct instruction *ip = t->ip;
7858
7859         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
7860
7861         JMP_CMP_MI(t, ip, >);
7862 }
7863
7864 static inline void
7865 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
7866 {
7867         struct thread *t = &p->threads[p->thread_id];
7868         struct instruction *ip = t->ip;
7869
7870         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
7871
7872         JMP_CMP_HI(t, ip, >);
7873 }
7874
7875 /*
7876  * return.
7877  */
7878 static int
7879 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
7880                        struct action *action,
7881                        char **tokens __rte_unused,
7882                        int n_tokens,
7883                        struct instruction *instr,
7884                        struct instruction_data *data __rte_unused)
7885 {
7886         CHECK(action, EINVAL);
7887         CHECK(n_tokens == 1, EINVAL);
7888
7889         instr->type = INSTR_RETURN;
7890         return 0;
7891 }
7892
7893 static inline void
7894 instr_return_exec(struct rte_swx_pipeline *p)
7895 {
7896         struct thread *t = &p->threads[p->thread_id];
7897
7898         TRACE("[Thread %2u] return\n", p->thread_id);
7899
7900         t->ip = t->ret;
7901 }
7902
7903 static int
7904 instr_translate(struct rte_swx_pipeline *p,
7905                 struct action *action,
7906                 char *string,
7907                 struct instruction *instr,
7908                 struct instruction_data *data)
7909 {
7910         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
7911         int n_tokens = 0, tpos = 0;
7912
7913         /* Parse the instruction string into tokens. */
7914         for ( ; ; ) {
7915                 char *token;
7916
7917                 token = strtok_r(string, " \t\v", &string);
7918                 if (!token)
7919                         break;
7920
7921                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
7922                 CHECK_NAME(token, EINVAL);
7923
7924                 tokens[n_tokens] = token;
7925                 n_tokens++;
7926         }
7927
7928         CHECK(n_tokens, EINVAL);
7929
7930         /* Handle the optional instruction label. */
7931         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
7932                 strcpy(data->label, tokens[0]);
7933
7934                 tpos += 2;
7935                 CHECK(n_tokens - tpos, EINVAL);
7936         }
7937
7938         /* Identify the instruction type. */
7939         if (!strcmp(tokens[tpos], "rx"))
7940                 return instr_rx_translate(p,
7941                                           action,
7942                                           &tokens[tpos],
7943                                           n_tokens - tpos,
7944                                           instr,
7945                                           data);
7946
7947         if (!strcmp(tokens[tpos], "tx"))
7948                 return instr_tx_translate(p,
7949                                           action,
7950                                           &tokens[tpos],
7951                                           n_tokens - tpos,
7952                                           instr,
7953                                           data);
7954
7955         if (!strcmp(tokens[tpos], "drop"))
7956                 return instr_drop_translate(p,
7957                                             action,
7958                                             &tokens[tpos],
7959                                             n_tokens - tpos,
7960                                             instr,
7961                                             data);
7962
7963         if (!strcmp(tokens[tpos], "extract"))
7964                 return instr_hdr_extract_translate(p,
7965                                                    action,
7966                                                    &tokens[tpos],
7967                                                    n_tokens - tpos,
7968                                                    instr,
7969                                                    data);
7970
7971         if (!strcmp(tokens[tpos], "lookahead"))
7972                 return instr_hdr_lookahead_translate(p,
7973                                                      action,
7974                                                      &tokens[tpos],
7975                                                      n_tokens - tpos,
7976                                                      instr,
7977                                                      data);
7978
7979         if (!strcmp(tokens[tpos], "emit"))
7980                 return instr_hdr_emit_translate(p,
7981                                                 action,
7982                                                 &tokens[tpos],
7983                                                 n_tokens - tpos,
7984                                                 instr,
7985                                                 data);
7986
7987         if (!strcmp(tokens[tpos], "validate"))
7988                 return instr_hdr_validate_translate(p,
7989                                                     action,
7990                                                     &tokens[tpos],
7991                                                     n_tokens - tpos,
7992                                                     instr,
7993                                                     data);
7994
7995         if (!strcmp(tokens[tpos], "invalidate"))
7996                 return instr_hdr_invalidate_translate(p,
7997                                                       action,
7998                                                       &tokens[tpos],
7999                                                       n_tokens - tpos,
8000                                                       instr,
8001                                                       data);
8002
8003         if (!strcmp(tokens[tpos], "mov"))
8004                 return instr_mov_translate(p,
8005                                            action,
8006                                            &tokens[tpos],
8007                                            n_tokens - tpos,
8008                                            instr,
8009                                            data);
8010
8011         if (!strcmp(tokens[tpos], "add"))
8012                 return instr_alu_add_translate(p,
8013                                                action,
8014                                                &tokens[tpos],
8015                                                n_tokens - tpos,
8016                                                instr,
8017                                                data);
8018
8019         if (!strcmp(tokens[tpos], "sub"))
8020                 return instr_alu_sub_translate(p,
8021                                                action,
8022                                                &tokens[tpos],
8023                                                n_tokens - tpos,
8024                                                instr,
8025                                                data);
8026
8027         if (!strcmp(tokens[tpos], "ckadd"))
8028                 return instr_alu_ckadd_translate(p,
8029                                                  action,
8030                                                  &tokens[tpos],
8031                                                  n_tokens - tpos,
8032                                                  instr,
8033                                                  data);
8034
8035         if (!strcmp(tokens[tpos], "cksub"))
8036                 return instr_alu_cksub_translate(p,
8037                                                  action,
8038                                                  &tokens[tpos],
8039                                                  n_tokens - tpos,
8040                                                  instr,
8041                                                  data);
8042
8043         if (!strcmp(tokens[tpos], "and"))
8044                 return instr_alu_and_translate(p,
8045                                                action,
8046                                                &tokens[tpos],
8047                                                n_tokens - tpos,
8048                                                instr,
8049                                                data);
8050
8051         if (!strcmp(tokens[tpos], "or"))
8052                 return instr_alu_or_translate(p,
8053                                               action,
8054                                               &tokens[tpos],
8055                                               n_tokens - tpos,
8056                                               instr,
8057                                               data);
8058
8059         if (!strcmp(tokens[tpos], "xor"))
8060                 return instr_alu_xor_translate(p,
8061                                                action,
8062                                                &tokens[tpos],
8063                                                n_tokens - tpos,
8064                                                instr,
8065                                                data);
8066
8067         if (!strcmp(tokens[tpos], "shl"))
8068                 return instr_alu_shl_translate(p,
8069                                                action,
8070                                                &tokens[tpos],
8071                                                n_tokens - tpos,
8072                                                instr,
8073                                                data);
8074
8075         if (!strcmp(tokens[tpos], "shr"))
8076                 return instr_alu_shr_translate(p,
8077                                                action,
8078                                                &tokens[tpos],
8079                                                n_tokens - tpos,
8080                                                instr,
8081                                                data);
8082
8083         if (!strcmp(tokens[tpos], "regprefetch"))
8084                 return instr_regprefetch_translate(p,
8085                                                    action,
8086                                                    &tokens[tpos],
8087                                                    n_tokens - tpos,
8088                                                    instr,
8089                                                    data);
8090
8091         if (!strcmp(tokens[tpos], "regrd"))
8092                 return instr_regrd_translate(p,
8093                                              action,
8094                                              &tokens[tpos],
8095                                              n_tokens - tpos,
8096                                              instr,
8097                                              data);
8098
8099         if (!strcmp(tokens[tpos], "regwr"))
8100                 return instr_regwr_translate(p,
8101                                              action,
8102                                              &tokens[tpos],
8103                                              n_tokens - tpos,
8104                                              instr,
8105                                              data);
8106
8107         if (!strcmp(tokens[tpos], "regadd"))
8108                 return instr_regadd_translate(p,
8109                                               action,
8110                                               &tokens[tpos],
8111                                               n_tokens - tpos,
8112                                               instr,
8113                                               data);
8114
8115         if (!strcmp(tokens[tpos], "metprefetch"))
8116                 return instr_metprefetch_translate(p,
8117                                                    action,
8118                                                    &tokens[tpos],
8119                                                    n_tokens - tpos,
8120                                                    instr,
8121                                                    data);
8122
8123         if (!strcmp(tokens[tpos], "meter"))
8124                 return instr_meter_translate(p,
8125                                              action,
8126                                              &tokens[tpos],
8127                                              n_tokens - tpos,
8128                                              instr,
8129                                              data);
8130
8131         if (!strcmp(tokens[tpos], "table"))
8132                 return instr_table_translate(p,
8133                                              action,
8134                                              &tokens[tpos],
8135                                              n_tokens - tpos,
8136                                              instr,
8137                                              data);
8138
8139         if (!strcmp(tokens[tpos], "extern"))
8140                 return instr_extern_translate(p,
8141                                               action,
8142                                               &tokens[tpos],
8143                                               n_tokens - tpos,
8144                                               instr,
8145                                               data);
8146
8147         if (!strcmp(tokens[tpos], "jmp"))
8148                 return instr_jmp_translate(p,
8149                                            action,
8150                                            &tokens[tpos],
8151                                            n_tokens - tpos,
8152                                            instr,
8153                                            data);
8154
8155         if (!strcmp(tokens[tpos], "jmpv"))
8156                 return instr_jmp_valid_translate(p,
8157                                                  action,
8158                                                  &tokens[tpos],
8159                                                  n_tokens - tpos,
8160                                                  instr,
8161                                                  data);
8162
8163         if (!strcmp(tokens[tpos], "jmpnv"))
8164                 return instr_jmp_invalid_translate(p,
8165                                                    action,
8166                                                    &tokens[tpos],
8167                                                    n_tokens - tpos,
8168                                                    instr,
8169                                                    data);
8170
8171         if (!strcmp(tokens[tpos], "jmph"))
8172                 return instr_jmp_hit_translate(p,
8173                                                action,
8174                                                &tokens[tpos],
8175                                                n_tokens - tpos,
8176                                                instr,
8177                                                data);
8178
8179         if (!strcmp(tokens[tpos], "jmpnh"))
8180                 return instr_jmp_miss_translate(p,
8181                                                 action,
8182                                                 &tokens[tpos],
8183                                                 n_tokens - tpos,
8184                                                 instr,
8185                                                 data);
8186
8187         if (!strcmp(tokens[tpos], "jmpa"))
8188                 return instr_jmp_action_hit_translate(p,
8189                                                       action,
8190                                                       &tokens[tpos],
8191                                                       n_tokens - tpos,
8192                                                       instr,
8193                                                       data);
8194
8195         if (!strcmp(tokens[tpos], "jmpna"))
8196                 return instr_jmp_action_miss_translate(p,
8197                                                        action,
8198                                                        &tokens[tpos],
8199                                                        n_tokens - tpos,
8200                                                        instr,
8201                                                        data);
8202
8203         if (!strcmp(tokens[tpos], "jmpeq"))
8204                 return instr_jmp_eq_translate(p,
8205                                               action,
8206                                               &tokens[tpos],
8207                                               n_tokens - tpos,
8208                                               instr,
8209                                               data);
8210
8211         if (!strcmp(tokens[tpos], "jmpneq"))
8212                 return instr_jmp_neq_translate(p,
8213                                                action,
8214                                                &tokens[tpos],
8215                                                n_tokens - tpos,
8216                                                instr,
8217                                                data);
8218
8219         if (!strcmp(tokens[tpos], "jmplt"))
8220                 return instr_jmp_lt_translate(p,
8221                                               action,
8222                                               &tokens[tpos],
8223                                               n_tokens - tpos,
8224                                               instr,
8225                                               data);
8226
8227         if (!strcmp(tokens[tpos], "jmpgt"))
8228                 return instr_jmp_gt_translate(p,
8229                                               action,
8230                                               &tokens[tpos],
8231                                               n_tokens - tpos,
8232                                               instr,
8233                                               data);
8234
8235         if (!strcmp(tokens[tpos], "return"))
8236                 return instr_return_translate(p,
8237                                               action,
8238                                               &tokens[tpos],
8239                                               n_tokens - tpos,
8240                                               instr,
8241                                               data);
8242
8243         CHECK(0, EINVAL);
8244 }
8245
8246 static struct instruction_data *
8247 label_find(struct instruction_data *data, uint32_t n, const char *label)
8248 {
8249         uint32_t i;
8250
8251         for (i = 0; i < n; i++)
8252                 if (!strcmp(label, data[i].label))
8253                         return &data[i];
8254
8255         return NULL;
8256 }
8257
8258 static uint32_t
8259 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
8260 {
8261         uint32_t count = 0, i;
8262
8263         if (!label[0])
8264                 return 0;
8265
8266         for (i = 0; i < n; i++)
8267                 if (!strcmp(label, data[i].jmp_label))
8268                         count++;
8269
8270         return count;
8271 }
8272
8273 static int
8274 instr_label_check(struct instruction_data *instruction_data,
8275                   uint32_t n_instructions)
8276 {
8277         uint32_t i;
8278
8279         /* Check that all instruction labels are unique. */
8280         for (i = 0; i < n_instructions; i++) {
8281                 struct instruction_data *data = &instruction_data[i];
8282                 char *label = data->label;
8283                 uint32_t j;
8284
8285                 if (!label[0])
8286                         continue;
8287
8288                 for (j = i + 1; j < n_instructions; j++)
8289                         CHECK(strcmp(label, data[j].label), EINVAL);
8290         }
8291
8292         /* Get users for each instruction label. */
8293         for (i = 0; i < n_instructions; i++) {
8294                 struct instruction_data *data = &instruction_data[i];
8295                 char *label = data->label;
8296
8297                 data->n_users = label_is_used(instruction_data,
8298                                               n_instructions,
8299                                               label);
8300         }
8301
8302         return 0;
8303 }
8304
8305 static int
8306 instr_jmp_resolve(struct instruction *instructions,
8307                   struct instruction_data *instruction_data,
8308                   uint32_t n_instructions)
8309 {
8310         uint32_t i;
8311
8312         for (i = 0; i < n_instructions; i++) {
8313                 struct instruction *instr = &instructions[i];
8314                 struct instruction_data *data = &instruction_data[i];
8315                 struct instruction_data *found;
8316
8317                 if (!instruction_is_jmp(instr))
8318                         continue;
8319
8320                 found = label_find(instruction_data,
8321                                    n_instructions,
8322                                    data->jmp_label);
8323                 CHECK(found, EINVAL);
8324
8325                 instr->jmp.ip = &instructions[found - instruction_data];
8326         }
8327
8328         return 0;
8329 }
8330
8331 static int
8332 instr_verify(struct rte_swx_pipeline *p __rte_unused,
8333              struct action *a,
8334              struct instruction *instr,
8335              struct instruction_data *data __rte_unused,
8336              uint32_t n_instructions)
8337 {
8338         if (!a) {
8339                 enum instruction_type type;
8340                 uint32_t i;
8341
8342                 /* Check that the first instruction is rx. */
8343                 CHECK(instr[0].type == INSTR_RX, EINVAL);
8344
8345                 /* Check that there is at least one tx instruction. */
8346                 for (i = 0; i < n_instructions; i++) {
8347                         type = instr[i].type;
8348
8349                         if (instruction_is_tx(type))
8350                                 break;
8351                 }
8352                 CHECK(i < n_instructions, EINVAL);
8353
8354                 /* Check that the last instruction is either tx or unconditional
8355                  * jump.
8356                  */
8357                 type = instr[n_instructions - 1].type;
8358                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
8359         }
8360
8361         if (a) {
8362                 enum instruction_type type;
8363                 uint32_t i;
8364
8365                 /* Check that there is at least one return or tx instruction. */
8366                 for (i = 0; i < n_instructions; i++) {
8367                         type = instr[i].type;
8368
8369                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
8370                                 break;
8371                 }
8372                 CHECK(i < n_instructions, EINVAL);
8373         }
8374
8375         return 0;
8376 }
8377
8378 static uint32_t
8379 instr_compact(struct instruction *instructions,
8380               struct instruction_data *instruction_data,
8381               uint32_t n_instructions)
8382 {
8383         uint32_t i, pos = 0;
8384
8385         /* Eliminate the invalid instructions that have been optimized out. */
8386         for (i = 0; i < n_instructions; i++) {
8387                 struct instruction *instr = &instructions[i];
8388                 struct instruction_data *data = &instruction_data[i];
8389
8390                 if (data->invalid)
8391                         continue;
8392
8393                 if (i != pos) {
8394                         memcpy(&instructions[pos], instr, sizeof(*instr));
8395                         memcpy(&instruction_data[pos], data, sizeof(*data));
8396                 }
8397
8398                 pos++;
8399         }
8400
8401         return pos;
8402 }
8403
8404 static int
8405 instr_pattern_extract_many_search(struct instruction *instr,
8406                                   struct instruction_data *data,
8407                                   uint32_t n_instr,
8408                                   uint32_t *n_pattern_instr)
8409 {
8410         uint32_t i;
8411
8412         for (i = 0; i < n_instr; i++) {
8413                 if (data[i].invalid)
8414                         break;
8415
8416                 if (instr[i].type != INSTR_HDR_EXTRACT)
8417                         break;
8418
8419                 if (i == RTE_DIM(instr->io.hdr.header_id))
8420                         break;
8421
8422                 if (i && data[i].n_users)
8423                         break;
8424         }
8425
8426         if (i < 2)
8427                 return 0;
8428
8429         *n_pattern_instr = i;
8430         return 1;
8431 }
8432
8433 static void
8434 instr_pattern_extract_many_replace(struct instruction *instr,
8435                                    struct instruction_data *data,
8436                                    uint32_t n_instr)
8437 {
8438         uint32_t i;
8439
8440         for (i = 1; i < n_instr; i++) {
8441                 instr[0].type++;
8442                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8443                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8444                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8445
8446                 data[i].invalid = 1;
8447         }
8448 }
8449
8450 static uint32_t
8451 instr_pattern_extract_many_optimize(struct instruction *instructions,
8452                                     struct instruction_data *instruction_data,
8453                                     uint32_t n_instructions)
8454 {
8455         uint32_t i;
8456
8457         for (i = 0; i < n_instructions; ) {
8458                 struct instruction *instr = &instructions[i];
8459                 struct instruction_data *data = &instruction_data[i];
8460                 uint32_t n_instr = 0;
8461                 int detected;
8462
8463                 /* Extract many. */
8464                 detected = instr_pattern_extract_many_search(instr,
8465                                                              data,
8466                                                              n_instructions - i,
8467                                                              &n_instr);
8468                 if (detected) {
8469                         instr_pattern_extract_many_replace(instr,
8470                                                            data,
8471                                                            n_instr);
8472                         i += n_instr;
8473                         continue;
8474                 }
8475
8476                 /* No pattern starting at the current instruction. */
8477                 i++;
8478         }
8479
8480         /* Eliminate the invalid instructions that have been optimized out. */
8481         n_instructions = instr_compact(instructions,
8482                                        instruction_data,
8483                                        n_instructions);
8484
8485         return n_instructions;
8486 }
8487
8488 static int
8489 instr_pattern_emit_many_tx_search(struct instruction *instr,
8490                                   struct instruction_data *data,
8491                                   uint32_t n_instr,
8492                                   uint32_t *n_pattern_instr)
8493 {
8494         uint32_t i;
8495
8496         for (i = 0; i < n_instr; i++) {
8497                 if (data[i].invalid)
8498                         break;
8499
8500                 if (instr[i].type != INSTR_HDR_EMIT)
8501                         break;
8502
8503                 if (i == RTE_DIM(instr->io.hdr.header_id))
8504                         break;
8505
8506                 if (i && data[i].n_users)
8507                         break;
8508         }
8509
8510         if (!i)
8511                 return 0;
8512
8513         if (!instruction_is_tx(instr[i].type))
8514                 return 0;
8515
8516         if (data[i].n_users)
8517                 return 0;
8518
8519         i++;
8520
8521         *n_pattern_instr = i;
8522         return 1;
8523 }
8524
8525 static void
8526 instr_pattern_emit_many_tx_replace(struct instruction *instr,
8527                                    struct instruction_data *data,
8528                                    uint32_t n_instr)
8529 {
8530         uint32_t i;
8531
8532         /* Any emit instruction in addition to the first one. */
8533         for (i = 1; i < n_instr - 1; i++) {
8534                 instr[0].type++;
8535                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8536                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8537                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8538
8539                 data[i].invalid = 1;
8540         }
8541
8542         /* The TX instruction is the last one in the pattern. */
8543         instr[0].type++;
8544         instr[0].io.io.offset = instr[i].io.io.offset;
8545         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
8546         data[i].invalid = 1;
8547 }
8548
8549 static uint32_t
8550 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
8551                                     struct instruction_data *instruction_data,
8552                                     uint32_t n_instructions)
8553 {
8554         uint32_t i;
8555
8556         for (i = 0; i < n_instructions; ) {
8557                 struct instruction *instr = &instructions[i];
8558                 struct instruction_data *data = &instruction_data[i];
8559                 uint32_t n_instr = 0;
8560                 int detected;
8561
8562                 /* Emit many + TX. */
8563                 detected = instr_pattern_emit_many_tx_search(instr,
8564                                                              data,
8565                                                              n_instructions - i,
8566                                                              &n_instr);
8567                 if (detected) {
8568                         instr_pattern_emit_many_tx_replace(instr,
8569                                                            data,
8570                                                            n_instr);
8571                         i += n_instr;
8572                         continue;
8573                 }
8574
8575                 /* No pattern starting at the current instruction. */
8576                 i++;
8577         }
8578
8579         /* Eliminate the invalid instructions that have been optimized out. */
8580         n_instructions = instr_compact(instructions,
8581                                        instruction_data,
8582                                        n_instructions);
8583
8584         return n_instructions;
8585 }
8586
8587 static uint32_t
8588 action_arg_src_mov_count(struct action *a,
8589                          uint32_t arg_id,
8590                          struct instruction *instructions,
8591                          struct instruction_data *instruction_data,
8592                          uint32_t n_instructions);
8593
8594 static int
8595 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
8596                                       struct action *a,
8597                                       struct instruction *instr,
8598                                       struct instruction_data *data,
8599                                       uint32_t n_instr,
8600                                       struct instruction *instructions,
8601                                       struct instruction_data *instruction_data,
8602                                       uint32_t n_instructions,
8603                                       uint32_t *n_pattern_instr)
8604 {
8605         struct header *h;
8606         uint32_t src_field_id, i, j;
8607
8608         /* Prerequisites. */
8609         if (!a || !a->st)
8610                 return 0;
8611
8612         /* First instruction: MOV_HM. */
8613         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
8614                 return 0;
8615
8616         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
8617         if (!h || h->st->var_size)
8618                 return 0;
8619
8620         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
8621                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
8622                         break;
8623
8624         if (src_field_id == a->st->n_fields)
8625                 return 0;
8626
8627         if (instr[0].mov.dst.offset ||
8628             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
8629             instr[0].mov.src.struct_id ||
8630             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
8631             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
8632                 return 0;
8633
8634         if ((n_instr < h->st->n_fields + 1) ||
8635              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
8636                 return 0;
8637
8638         /* Subsequent instructions: MOV_HM. */
8639         for (i = 1; i < h->st->n_fields; i++)
8640                 if (data[i].invalid ||
8641                     data[i].n_users ||
8642                     (instr[i].type != INSTR_MOV_HM) ||
8643                     (instr[i].mov.dst.struct_id != h->struct_id) ||
8644                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
8645                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
8646                     instr[i].mov.src.struct_id ||
8647                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
8648                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
8649                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
8650                         return 0;
8651
8652         /* Last instruction: HDR_VALIDATE. */
8653         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
8654             (instr[i].valid.header_id != h->id))
8655                 return 0;
8656
8657         /* Check that none of the action args that are used as source for this
8658          * DMA transfer are not used as source in any other mov instruction.
8659          */
8660         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
8661                 uint32_t n_users;
8662
8663                 n_users = action_arg_src_mov_count(a,
8664                                                    j,
8665                                                    instructions,
8666                                                    instruction_data,
8667                                                    n_instructions);
8668                 if (n_users > 1)
8669                         return 0;
8670         }
8671
8672         *n_pattern_instr = 1 + i;
8673         return 1;
8674 }
8675
8676 static void
8677 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
8678                                        struct action *a,
8679                                        struct instruction *instr,
8680                                        struct instruction_data *data,
8681                                        uint32_t n_instr)
8682 {
8683         struct header *h;
8684         uint32_t src_field_id, src_offset, i;
8685
8686         /* Read from the instructions before they are modified. */
8687         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
8688         if (!h)
8689                 return;
8690
8691         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
8692                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
8693                         break;
8694
8695         if (src_field_id == a->st->n_fields)
8696                 return;
8697
8698         src_offset = instr[0].mov.src.offset;
8699
8700         /* Modify the instructions. */
8701         instr[0].type = INSTR_DMA_HT;
8702         instr[0].dma.dst.header_id[0] = h->id;
8703         instr[0].dma.dst.struct_id[0] = h->struct_id;
8704         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
8705         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
8706
8707         for (i = 1; i < n_instr; i++)
8708                 data[i].invalid = 1;
8709
8710         /* Update the endianness of the action arguments to header endianness. */
8711         for (i = 0; i < h->st->n_fields; i++)
8712                 a->args_endianness[src_field_id + i] = 1;
8713 }
8714
8715 static uint32_t
8716 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
8717                                         struct action *a,
8718                                         struct instruction *instructions,
8719                                         struct instruction_data *instruction_data,
8720                                         uint32_t n_instructions)
8721 {
8722         uint32_t i;
8723
8724         if (!a || !a->st)
8725                 return n_instructions;
8726
8727         for (i = 0; i < n_instructions; ) {
8728                 struct instruction *instr = &instructions[i];
8729                 struct instruction_data *data = &instruction_data[i];
8730                 uint32_t n_instr = 0;
8731                 int detected;
8732
8733                 /* Mov all + validate. */
8734                 detected = instr_pattern_mov_all_validate_search(p,
8735                                                                  a,
8736                                                                  instr,
8737                                                                  data,
8738                                                                  n_instructions - i,
8739                                                                  instructions,
8740                                                                  instruction_data,
8741                                                                  n_instructions,
8742                                                                  &n_instr);
8743                 if (detected) {
8744                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
8745                         i += n_instr;
8746                         continue;
8747                 }
8748
8749                 /* No pattern starting at the current instruction. */
8750                 i++;
8751         }
8752
8753         /* Eliminate the invalid instructions that have been optimized out. */
8754         n_instructions = instr_compact(instructions,
8755                                        instruction_data,
8756                                        n_instructions);
8757
8758         return n_instructions;
8759 }
8760
8761 static int
8762 instr_pattern_dma_many_search(struct instruction *instr,
8763                               struct instruction_data *data,
8764                               uint32_t n_instr,
8765                               uint32_t *n_pattern_instr)
8766 {
8767         uint32_t i;
8768
8769         for (i = 0; i < n_instr; i++) {
8770                 if (data[i].invalid)
8771                         break;
8772
8773                 if (instr[i].type != INSTR_DMA_HT)
8774                         break;
8775
8776                 if (i == RTE_DIM(instr->dma.dst.header_id))
8777                         break;
8778
8779                 if (i && data[i].n_users)
8780                         break;
8781         }
8782
8783         if (i < 2)
8784                 return 0;
8785
8786         *n_pattern_instr = i;
8787         return 1;
8788 }
8789
8790 static void
8791 instr_pattern_dma_many_replace(struct instruction *instr,
8792                                struct instruction_data *data,
8793                                uint32_t n_instr)
8794 {
8795         uint32_t i;
8796
8797         for (i = 1; i < n_instr; i++) {
8798                 instr[0].type++;
8799                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
8800                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
8801                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
8802                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
8803
8804                 data[i].invalid = 1;
8805         }
8806 }
8807
8808 static uint32_t
8809 instr_pattern_dma_many_optimize(struct instruction *instructions,
8810                struct instruction_data *instruction_data,
8811                uint32_t n_instructions)
8812 {
8813         uint32_t i;
8814
8815         for (i = 0; i < n_instructions; ) {
8816                 struct instruction *instr = &instructions[i];
8817                 struct instruction_data *data = &instruction_data[i];
8818                 uint32_t n_instr = 0;
8819                 int detected;
8820
8821                 /* DMA many. */
8822                 detected = instr_pattern_dma_many_search(instr,
8823                                                          data,
8824                                                          n_instructions - i,
8825                                                          &n_instr);
8826                 if (detected) {
8827                         instr_pattern_dma_many_replace(instr, data, n_instr);
8828                         i += n_instr;
8829                         continue;
8830                 }
8831
8832                 /* No pattern starting at the current instruction. */
8833                 i++;
8834         }
8835
8836         /* Eliminate the invalid instructions that have been optimized out. */
8837         n_instructions = instr_compact(instructions,
8838                                        instruction_data,
8839                                        n_instructions);
8840
8841         return n_instructions;
8842 }
8843
8844 static uint32_t
8845 instr_optimize(struct rte_swx_pipeline *p,
8846                struct action *a,
8847                struct instruction *instructions,
8848                struct instruction_data *instruction_data,
8849                uint32_t n_instructions)
8850 {
8851         /* Extract many. */
8852         n_instructions = instr_pattern_extract_many_optimize(instructions,
8853                                                              instruction_data,
8854                                                              n_instructions);
8855
8856         /* Emit many + TX. */
8857         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
8858                                                              instruction_data,
8859                                                              n_instructions);
8860
8861         /* Mov all + validate. */
8862         n_instructions = instr_pattern_mov_all_validate_optimize(p,
8863                                                                  a,
8864                                                                  instructions,
8865                                                                  instruction_data,
8866                                                                  n_instructions);
8867
8868         /* DMA many. */
8869         n_instructions = instr_pattern_dma_many_optimize(instructions,
8870                                                          instruction_data,
8871                                                          n_instructions);
8872
8873         return n_instructions;
8874 }
8875
8876 static int
8877 instruction_config(struct rte_swx_pipeline *p,
8878                    struct action *a,
8879                    const char **instructions,
8880                    uint32_t n_instructions)
8881 {
8882         struct instruction *instr = NULL;
8883         struct instruction_data *data = NULL;
8884         int err = 0;
8885         uint32_t i;
8886
8887         CHECK(n_instructions, EINVAL);
8888         CHECK(instructions, EINVAL);
8889         for (i = 0; i < n_instructions; i++)
8890                 CHECK_INSTRUCTION(instructions[i], EINVAL);
8891
8892         /* Memory allocation. */
8893         instr = calloc(n_instructions, sizeof(struct instruction));
8894         if (!instr) {
8895                 err = -ENOMEM;
8896                 goto error;
8897         }
8898
8899         data = calloc(n_instructions, sizeof(struct instruction_data));
8900         if (!data) {
8901                 err = -ENOMEM;
8902                 goto error;
8903         }
8904
8905         for (i = 0; i < n_instructions; i++) {
8906                 char *string = strdup(instructions[i]);
8907                 if (!string) {
8908                         err = -ENOMEM;
8909                         goto error;
8910                 }
8911
8912                 err = instr_translate(p, a, string, &instr[i], &data[i]);
8913                 if (err) {
8914                         free(string);
8915                         goto error;
8916                 }
8917
8918                 free(string);
8919         }
8920
8921         err = instr_label_check(data, n_instructions);
8922         if (err)
8923                 goto error;
8924
8925         err = instr_verify(p, a, instr, data, n_instructions);
8926         if (err)
8927                 goto error;
8928
8929         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
8930
8931         err = instr_jmp_resolve(instr, data, n_instructions);
8932         if (err)
8933                 goto error;
8934
8935         if (a) {
8936                 a->instructions = instr;
8937                 a->n_instructions = n_instructions;
8938         } else {
8939                 p->instructions = instr;
8940                 p->n_instructions = n_instructions;
8941         }
8942
8943         free(data);
8944         return 0;
8945
8946 error:
8947         free(data);
8948         free(instr);
8949         return err;
8950 }
8951
8952 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
8953
8954 static instr_exec_t instruction_table[] = {
8955         [INSTR_RX] = instr_rx_exec,
8956         [INSTR_TX] = instr_tx_exec,
8957         [INSTR_TX_I] = instr_tx_i_exec,
8958
8959         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
8960         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
8961         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
8962         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
8963         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
8964         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
8965         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
8966         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
8967         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
8968         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
8969
8970         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
8971         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
8972         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
8973         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
8974         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
8975         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
8976         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
8977         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
8978         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
8979
8980         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
8981         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
8982
8983         [INSTR_MOV] = instr_mov_exec,
8984         [INSTR_MOV_MH] = instr_mov_mh_exec,
8985         [INSTR_MOV_HM] = instr_mov_hm_exec,
8986         [INSTR_MOV_HH] = instr_mov_hh_exec,
8987         [INSTR_MOV_I] = instr_mov_i_exec,
8988
8989         [INSTR_DMA_HT] = instr_dma_ht_exec,
8990         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
8991         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
8992         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
8993         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
8994         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
8995         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
8996         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
8997
8998         [INSTR_ALU_ADD] = instr_alu_add_exec,
8999         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
9000         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
9001         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
9002         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
9003         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
9004
9005         [INSTR_ALU_SUB] = instr_alu_sub_exec,
9006         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
9007         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
9008         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
9009         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
9010         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
9011
9012         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
9013         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
9014         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
9015         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
9016
9017         [INSTR_ALU_AND] = instr_alu_and_exec,
9018         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
9019         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
9020         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
9021         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
9022
9023         [INSTR_ALU_OR] = instr_alu_or_exec,
9024         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
9025         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
9026         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
9027         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
9028
9029         [INSTR_ALU_XOR] = instr_alu_xor_exec,
9030         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
9031         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
9032         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
9033         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
9034
9035         [INSTR_ALU_SHL] = instr_alu_shl_exec,
9036         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
9037         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
9038         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
9039         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
9040         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
9041
9042         [INSTR_ALU_SHR] = instr_alu_shr_exec,
9043         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
9044         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
9045         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
9046         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
9047         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
9048
9049         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
9050         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
9051         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
9052
9053         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
9054         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
9055         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
9056         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
9057         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
9058         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
9059
9060         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
9061         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
9062         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
9063         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
9064         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
9065         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
9066         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
9067         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
9068         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
9069
9070         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
9071         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
9072         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
9073         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
9074         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
9075         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
9076         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
9077         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
9078         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
9079
9080         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
9081         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
9082         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
9083
9084         [INSTR_METER_HHM] = instr_meter_hhm_exec,
9085         [INSTR_METER_HHI] = instr_meter_hhi_exec,
9086         [INSTR_METER_HMM] = instr_meter_hmm_exec,
9087         [INSTR_METER_HMI] = instr_meter_hmi_exec,
9088         [INSTR_METER_MHM] = instr_meter_mhm_exec,
9089         [INSTR_METER_MHI] = instr_meter_mhi_exec,
9090         [INSTR_METER_MMM] = instr_meter_mmm_exec,
9091         [INSTR_METER_MMI] = instr_meter_mmi_exec,
9092         [INSTR_METER_IHM] = instr_meter_ihm_exec,
9093         [INSTR_METER_IHI] = instr_meter_ihi_exec,
9094         [INSTR_METER_IMM] = instr_meter_imm_exec,
9095         [INSTR_METER_IMI] = instr_meter_imi_exec,
9096
9097         [INSTR_TABLE] = instr_table_exec,
9098         [INSTR_SELECTOR] = instr_selector_exec,
9099         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
9100         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
9101
9102         [INSTR_JMP] = instr_jmp_exec,
9103         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
9104         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
9105         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
9106         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
9107         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
9108         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
9109
9110         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
9111         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
9112         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
9113         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
9114         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
9115
9116         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
9117         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
9118         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
9119         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
9120         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
9121
9122         [INSTR_JMP_LT] = instr_jmp_lt_exec,
9123         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
9124         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
9125         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
9126         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
9127         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
9128
9129         [INSTR_JMP_GT] = instr_jmp_gt_exec,
9130         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
9131         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
9132         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
9133         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
9134         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
9135
9136         [INSTR_RETURN] = instr_return_exec,
9137 };
9138
9139 static inline void
9140 instr_exec(struct rte_swx_pipeline *p)
9141 {
9142         struct thread *t = &p->threads[p->thread_id];
9143         struct instruction *ip = t->ip;
9144         instr_exec_t instr = instruction_table[ip->type];
9145
9146         instr(p);
9147 }
9148
9149 /*
9150  * Action.
9151  */
9152 static struct action *
9153 action_find(struct rte_swx_pipeline *p, const char *name)
9154 {
9155         struct action *elem;
9156
9157         if (!name)
9158                 return NULL;
9159
9160         TAILQ_FOREACH(elem, &p->actions, node)
9161                 if (strcmp(elem->name, name) == 0)
9162                         return elem;
9163
9164         return NULL;
9165 }
9166
9167 static struct action *
9168 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9169 {
9170         struct action *action = NULL;
9171
9172         TAILQ_FOREACH(action, &p->actions, node)
9173                 if (action->id == id)
9174                         return action;
9175
9176         return NULL;
9177 }
9178
9179 static struct field *
9180 action_field_find(struct action *a, const char *name)
9181 {
9182         return a->st ? struct_type_field_find(a->st, name) : NULL;
9183 }
9184
9185 static struct field *
9186 action_field_parse(struct action *action, const char *name)
9187 {
9188         if (name[0] != 't' || name[1] != '.')
9189                 return NULL;
9190
9191         return action_field_find(action, &name[2]);
9192 }
9193
9194 int
9195 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
9196                                const char *name,
9197                                const char *args_struct_type_name,
9198                                const char **instructions,
9199                                uint32_t n_instructions)
9200 {
9201         struct struct_type *args_struct_type = NULL;
9202         struct action *a;
9203         int err;
9204
9205         CHECK(p, EINVAL);
9206
9207         CHECK_NAME(name, EINVAL);
9208         CHECK(!action_find(p, name), EEXIST);
9209
9210         if (args_struct_type_name) {
9211                 CHECK_NAME(args_struct_type_name, EINVAL);
9212                 args_struct_type = struct_type_find(p, args_struct_type_name);
9213                 CHECK(args_struct_type, EINVAL);
9214                 CHECK(!args_struct_type->var_size, EINVAL);
9215         }
9216
9217         /* Node allocation. */
9218         a = calloc(1, sizeof(struct action));
9219         CHECK(a, ENOMEM);
9220         if (args_struct_type) {
9221                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
9222                 if (!a->args_endianness) {
9223                         free(a);
9224                         CHECK(0, ENOMEM);
9225                 }
9226         }
9227
9228         /* Node initialization. */
9229         strcpy(a->name, name);
9230         a->st = args_struct_type;
9231         a->id = p->n_actions;
9232
9233         /* Instruction translation. */
9234         err = instruction_config(p, a, instructions, n_instructions);
9235         if (err) {
9236                 free(a->args_endianness);
9237                 free(a);
9238                 return err;
9239         }
9240
9241         /* Node add to tailq. */
9242         TAILQ_INSERT_TAIL(&p->actions, a, node);
9243         p->n_actions++;
9244
9245         return 0;
9246 }
9247
9248 static int
9249 action_build(struct rte_swx_pipeline *p)
9250 {
9251         struct action *action;
9252
9253         p->action_instructions = calloc(p->n_actions,
9254                                         sizeof(struct instruction *));
9255         CHECK(p->action_instructions, ENOMEM);
9256
9257         TAILQ_FOREACH(action, &p->actions, node)
9258                 p->action_instructions[action->id] = action->instructions;
9259
9260         return 0;
9261 }
9262
9263 static void
9264 action_build_free(struct rte_swx_pipeline *p)
9265 {
9266         free(p->action_instructions);
9267         p->action_instructions = NULL;
9268 }
9269
9270 static void
9271 action_free(struct rte_swx_pipeline *p)
9272 {
9273         action_build_free(p);
9274
9275         for ( ; ; ) {
9276                 struct action *action;
9277
9278                 action = TAILQ_FIRST(&p->actions);
9279                 if (!action)
9280                         break;
9281
9282                 TAILQ_REMOVE(&p->actions, action, node);
9283                 free(action->instructions);
9284                 free(action);
9285         }
9286 }
9287
9288 static uint32_t
9289 action_arg_src_mov_count(struct action *a,
9290                          uint32_t arg_id,
9291                          struct instruction *instructions,
9292                          struct instruction_data *instruction_data,
9293                          uint32_t n_instructions)
9294 {
9295         uint32_t offset, n_users = 0, i;
9296
9297         if (!a->st ||
9298             (arg_id >= a->st->n_fields) ||
9299             !instructions ||
9300             !instruction_data ||
9301             !n_instructions)
9302                 return 0;
9303
9304         offset = a->st->fields[arg_id].offset / 8;
9305
9306         for (i = 0; i < n_instructions; i++) {
9307                 struct instruction *instr = &instructions[i];
9308                 struct instruction_data *data = &instruction_data[i];
9309
9310                 if (data->invalid ||
9311                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
9312                     instr->mov.src.struct_id ||
9313                     (instr->mov.src.offset != offset))
9314                         continue;
9315
9316                 n_users++;
9317         }
9318
9319         return n_users;
9320 }
9321
9322 /*
9323  * Table.
9324  */
9325 static struct table_type *
9326 table_type_find(struct rte_swx_pipeline *p, const char *name)
9327 {
9328         struct table_type *elem;
9329
9330         TAILQ_FOREACH(elem, &p->table_types, node)
9331                 if (strcmp(elem->name, name) == 0)
9332                         return elem;
9333
9334         return NULL;
9335 }
9336
9337 static struct table_type *
9338 table_type_resolve(struct rte_swx_pipeline *p,
9339                    const char *recommended_type_name,
9340                    enum rte_swx_table_match_type match_type)
9341 {
9342         struct table_type *elem;
9343
9344         /* Only consider the recommended type if the match type is correct. */
9345         if (recommended_type_name)
9346                 TAILQ_FOREACH(elem, &p->table_types, node)
9347                         if (!strcmp(elem->name, recommended_type_name) &&
9348                             (elem->match_type == match_type))
9349                                 return elem;
9350
9351         /* Ignore the recommended type and get the first element with this match
9352          * type.
9353          */
9354         TAILQ_FOREACH(elem, &p->table_types, node)
9355                 if (elem->match_type == match_type)
9356                         return elem;
9357
9358         return NULL;
9359 }
9360
9361 static struct table *
9362 table_find(struct rte_swx_pipeline *p, const char *name)
9363 {
9364         struct table *elem;
9365
9366         TAILQ_FOREACH(elem, &p->tables, node)
9367                 if (strcmp(elem->name, name) == 0)
9368                         return elem;
9369
9370         return NULL;
9371 }
9372
9373 static struct table *
9374 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9375 {
9376         struct table *table = NULL;
9377
9378         TAILQ_FOREACH(table, &p->tables, node)
9379                 if (table->id == id)
9380                         return table;
9381
9382         return NULL;
9383 }
9384
9385 int
9386 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
9387                                      const char *name,
9388                                      enum rte_swx_table_match_type match_type,
9389                                      struct rte_swx_table_ops *ops)
9390 {
9391         struct table_type *elem;
9392
9393         CHECK(p, EINVAL);
9394
9395         CHECK_NAME(name, EINVAL);
9396         CHECK(!table_type_find(p, name), EEXIST);
9397
9398         CHECK(ops, EINVAL);
9399         CHECK(ops->create, EINVAL);
9400         CHECK(ops->lkp, EINVAL);
9401         CHECK(ops->free, EINVAL);
9402
9403         /* Node allocation. */
9404         elem = calloc(1, sizeof(struct table_type));
9405         CHECK(elem, ENOMEM);
9406
9407         /* Node initialization. */
9408         strcpy(elem->name, name);
9409         elem->match_type = match_type;
9410         memcpy(&elem->ops, ops, sizeof(*ops));
9411
9412         /* Node add to tailq. */
9413         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
9414
9415         return 0;
9416 }
9417
9418 static int
9419 table_match_type_resolve(struct rte_swx_match_field_params *fields,
9420                          uint32_t n_fields,
9421                          enum rte_swx_table_match_type *match_type)
9422 {
9423         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
9424
9425         for (i = 0; i < n_fields; i++) {
9426                 struct rte_swx_match_field_params  *f = &fields[i];
9427
9428                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
9429                         n_fields_em++;
9430
9431                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
9432                         n_fields_lpm++;
9433         }
9434
9435         if ((n_fields_lpm > 1) ||
9436             (n_fields_lpm && (n_fields_em != n_fields - 1)))
9437                 return -EINVAL;
9438
9439         *match_type = (n_fields_em == n_fields) ?
9440                        RTE_SWX_TABLE_MATCH_EXACT :
9441                        RTE_SWX_TABLE_MATCH_WILDCARD;
9442
9443         return 0;
9444 }
9445
9446 static int
9447 table_match_fields_check(struct rte_swx_pipeline *p,
9448                          struct rte_swx_pipeline_table_params *params,
9449                          struct header **header)
9450 {
9451         struct header *h0 = NULL;
9452         struct field *hf, *mf;
9453         uint32_t *offset = NULL, i;
9454         int status = 0;
9455
9456         /* Return if no match fields. */
9457         if (!params->n_fields) {
9458                 if (params->fields) {
9459                         status = -EINVAL;
9460                         goto end;
9461                 }
9462
9463                 if (header)
9464                         *header = NULL;
9465
9466                 return 0;
9467         }
9468
9469         /* Memory allocation. */
9470         offset = calloc(params->n_fields, sizeof(uint32_t));
9471         if (!offset) {
9472                 status = -ENOMEM;
9473                 goto end;
9474         }
9475
9476         /* Check that all the match fields belong to either the same header or
9477          * to the meta-data.
9478          */
9479         hf = header_field_parse(p, params->fields[0].name, &h0);
9480         mf = metadata_field_parse(p, params->fields[0].name);
9481         if ((!hf && !mf) || (hf && hf->var_size)) {
9482                 status = -EINVAL;
9483                 goto end;
9484         }
9485
9486         offset[0] = h0 ? hf->offset : mf->offset;
9487
9488         for (i = 1; i < params->n_fields; i++)
9489                 if (h0) {
9490                         struct header *h;
9491
9492                         hf = header_field_parse(p, params->fields[i].name, &h);
9493                         if (!hf || (h->id != h0->id) || hf->var_size) {
9494                                 status = -EINVAL;
9495                                 goto end;
9496                         }
9497
9498                         offset[i] = hf->offset;
9499                 } else {
9500                         mf = metadata_field_parse(p, params->fields[i].name);
9501                         if (!mf) {
9502                                 status = -EINVAL;
9503                                 goto end;
9504                         }
9505
9506                         offset[i] = mf->offset;
9507                 }
9508
9509         /* Check that there are no duplicated match fields. */
9510         for (i = 0; i < params->n_fields; i++) {
9511                 uint32_t j;
9512
9513                 for (j = 0; j < i; j++)
9514                         if (offset[j] == offset[i]) {
9515                                 status = -EINVAL;
9516                                 goto end;
9517                         }
9518         }
9519
9520         /* Return. */
9521         if (header)
9522                 *header = h0;
9523
9524 end:
9525         free(offset);
9526         return status;
9527 }
9528
9529 int
9530 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
9531                               const char *name,
9532                               struct rte_swx_pipeline_table_params *params,
9533                               const char *recommended_table_type_name,
9534                               const char *args,
9535                               uint32_t size)
9536 {
9537         struct table_type *type;
9538         struct table *t;
9539         struct action *default_action;
9540         struct header *header = NULL;
9541         uint32_t action_data_size_max = 0, i;
9542         int status = 0;
9543
9544         CHECK(p, EINVAL);
9545
9546         CHECK_NAME(name, EINVAL);
9547         CHECK(!table_find(p, name), EEXIST);
9548         CHECK(!selector_find(p, name), EEXIST);
9549
9550         CHECK(params, EINVAL);
9551
9552         /* Match checks. */
9553         status = table_match_fields_check(p, params, &header);
9554         if (status)
9555                 return status;
9556
9557         /* Action checks. */
9558         CHECK(params->n_actions, EINVAL);
9559         CHECK(params->action_names, EINVAL);
9560         for (i = 0; i < params->n_actions; i++) {
9561                 const char *action_name = params->action_names[i];
9562                 struct action *a;
9563                 uint32_t action_data_size;
9564
9565                 CHECK_NAME(action_name, EINVAL);
9566
9567                 a = action_find(p, action_name);
9568                 CHECK(a, EINVAL);
9569
9570                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
9571                 if (action_data_size > action_data_size_max)
9572                         action_data_size_max = action_data_size;
9573         }
9574
9575         CHECK_NAME(params->default_action_name, EINVAL);
9576         for (i = 0; i < p->n_actions; i++)
9577                 if (!strcmp(params->action_names[i],
9578                             params->default_action_name))
9579                         break;
9580         CHECK(i < params->n_actions, EINVAL);
9581         default_action = action_find(p, params->default_action_name);
9582         CHECK((default_action->st && params->default_action_data) ||
9583               !params->default_action_data, EINVAL);
9584
9585         /* Table type checks. */
9586         if (recommended_table_type_name)
9587                 CHECK_NAME(recommended_table_type_name, EINVAL);
9588
9589         if (params->n_fields) {
9590                 enum rte_swx_table_match_type match_type;
9591
9592                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
9593                 if (status)
9594                         return status;
9595
9596                 type = table_type_resolve(p, recommended_table_type_name, match_type);
9597                 CHECK(type, EINVAL);
9598         } else {
9599                 type = NULL;
9600         }
9601
9602         /* Memory allocation. */
9603         t = calloc(1, sizeof(struct table));
9604         CHECK(t, ENOMEM);
9605
9606         t->fields = calloc(params->n_fields, sizeof(struct match_field));
9607         if (!t->fields) {
9608                 free(t);
9609                 CHECK(0, ENOMEM);
9610         }
9611
9612         t->actions = calloc(params->n_actions, sizeof(struct action *));
9613         if (!t->actions) {
9614                 free(t->fields);
9615                 free(t);
9616                 CHECK(0, ENOMEM);
9617         }
9618
9619         if (action_data_size_max) {
9620                 t->default_action_data = calloc(1, action_data_size_max);
9621                 if (!t->default_action_data) {
9622                         free(t->actions);
9623                         free(t->fields);
9624                         free(t);
9625                         CHECK(0, ENOMEM);
9626                 }
9627         }
9628
9629         /* Node initialization. */
9630         strcpy(t->name, name);
9631         if (args && args[0])
9632                 strcpy(t->args, args);
9633         t->type = type;
9634
9635         for (i = 0; i < params->n_fields; i++) {
9636                 struct rte_swx_match_field_params *field = &params->fields[i];
9637                 struct match_field *f = &t->fields[i];
9638
9639                 f->match_type = field->match_type;
9640                 f->field = header ?
9641                         header_field_parse(p, field->name, NULL) :
9642                         metadata_field_parse(p, field->name);
9643         }
9644         t->n_fields = params->n_fields;
9645         t->header = header;
9646
9647         for (i = 0; i < params->n_actions; i++)
9648                 t->actions[i] = action_find(p, params->action_names[i]);
9649         t->default_action = default_action;
9650         if (default_action->st)
9651                 memcpy(t->default_action_data,
9652                        params->default_action_data,
9653                        default_action->st->n_bits / 8);
9654         t->n_actions = params->n_actions;
9655         t->default_action_is_const = params->default_action_is_const;
9656         t->action_data_size_max = action_data_size_max;
9657
9658         t->size = size;
9659         t->id = p->n_tables;
9660
9661         /* Node add to tailq. */
9662         TAILQ_INSERT_TAIL(&p->tables, t, node);
9663         p->n_tables++;
9664
9665         return 0;
9666 }
9667
9668 static struct rte_swx_table_params *
9669 table_params_get(struct table *table)
9670 {
9671         struct rte_swx_table_params *params;
9672         struct field *first, *last;
9673         uint8_t *key_mask;
9674         uint32_t key_size, key_offset, action_data_size, i;
9675
9676         /* Memory allocation. */
9677         params = calloc(1, sizeof(struct rte_swx_table_params));
9678         if (!params)
9679                 return NULL;
9680
9681         /* Find first (smallest offset) and last (biggest offset) match fields. */
9682         first = table->fields[0].field;
9683         last = table->fields[0].field;
9684
9685         for (i = 0; i < table->n_fields; i++) {
9686                 struct field *f = table->fields[i].field;
9687
9688                 if (f->offset < first->offset)
9689                         first = f;
9690
9691                 if (f->offset > last->offset)
9692                         last = f;
9693         }
9694
9695         /* Key offset and size. */
9696         key_offset = first->offset / 8;
9697         key_size = (last->offset + last->n_bits - first->offset) / 8;
9698
9699         /* Memory allocation. */
9700         key_mask = calloc(1, key_size);
9701         if (!key_mask) {
9702                 free(params);
9703                 return NULL;
9704         }
9705
9706         /* Key mask. */
9707         for (i = 0; i < table->n_fields; i++) {
9708                 struct field *f = table->fields[i].field;
9709                 uint32_t start = (f->offset - first->offset) / 8;
9710                 size_t size = f->n_bits / 8;
9711
9712                 memset(&key_mask[start], 0xFF, size);
9713         }
9714
9715         /* Action data size. */
9716         action_data_size = 0;
9717         for (i = 0; i < table->n_actions; i++) {
9718                 struct action *action = table->actions[i];
9719                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
9720
9721                 if (ads > action_data_size)
9722                         action_data_size = ads;
9723         }
9724
9725         /* Fill in. */
9726         params->match_type = table->type->match_type;
9727         params->key_size = key_size;
9728         params->key_offset = key_offset;
9729         params->key_mask0 = key_mask;
9730         params->action_data_size = action_data_size;
9731         params->n_keys_max = table->size;
9732
9733         return params;
9734 }
9735
9736 static void
9737 table_params_free(struct rte_swx_table_params *params)
9738 {
9739         if (!params)
9740                 return;
9741
9742         free(params->key_mask0);
9743         free(params);
9744 }
9745
9746 static int
9747 table_stub_lkp(void *table __rte_unused,
9748                void *mailbox __rte_unused,
9749                uint8_t **key __rte_unused,
9750                uint64_t *action_id __rte_unused,
9751                uint8_t **action_data __rte_unused,
9752                int *hit)
9753 {
9754         *hit = 0;
9755         return 1; /* DONE. */
9756 }
9757
9758 static int
9759 table_build(struct rte_swx_pipeline *p)
9760 {
9761         uint32_t i;
9762
9763         /* Per pipeline: table statistics. */
9764         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
9765         CHECK(p->table_stats, ENOMEM);
9766
9767         for (i = 0; i < p->n_tables; i++) {
9768                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9769                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
9770         }
9771
9772         /* Per thread: table runt-time. */
9773         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9774                 struct thread *t = &p->threads[i];
9775                 struct table *table;
9776
9777                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
9778                 CHECK(t->tables, ENOMEM);
9779
9780                 TAILQ_FOREACH(table, &p->tables, node) {
9781                         struct table_runtime *r = &t->tables[table->id];
9782
9783                         if (table->type) {
9784                                 uint64_t size;
9785
9786                                 size = table->type->ops.mailbox_size_get();
9787
9788                                 /* r->func. */
9789                                 r->func = table->type->ops.lkp;
9790
9791                                 /* r->mailbox. */
9792                                 if (size) {
9793                                         r->mailbox = calloc(1, size);
9794                                         CHECK(r->mailbox, ENOMEM);
9795                                 }
9796
9797                                 /* r->key. */
9798                                 r->key = table->header ?
9799                                         &t->structs[table->header->struct_id] :
9800                                         &t->structs[p->metadata_struct_id];
9801                         } else {
9802                                 r->func = table_stub_lkp;
9803                         }
9804                 }
9805         }
9806
9807         return 0;
9808 }
9809
9810 static void
9811 table_build_free(struct rte_swx_pipeline *p)
9812 {
9813         uint32_t i;
9814
9815         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9816                 struct thread *t = &p->threads[i];
9817                 uint32_t j;
9818
9819                 if (!t->tables)
9820                         continue;
9821
9822                 for (j = 0; j < p->n_tables; j++) {
9823                         struct table_runtime *r = &t->tables[j];
9824
9825                         free(r->mailbox);
9826                 }
9827
9828                 free(t->tables);
9829                 t->tables = NULL;
9830         }
9831
9832         if (p->table_stats) {
9833                 for (i = 0; i < p->n_tables; i++)
9834                         free(p->table_stats[i].n_pkts_action);
9835
9836                 free(p->table_stats);
9837         }
9838 }
9839
9840 static void
9841 table_free(struct rte_swx_pipeline *p)
9842 {
9843         table_build_free(p);
9844
9845         /* Tables. */
9846         for ( ; ; ) {
9847                 struct table *elem;
9848
9849                 elem = TAILQ_FIRST(&p->tables);
9850                 if (!elem)
9851                         break;
9852
9853                 TAILQ_REMOVE(&p->tables, elem, node);
9854                 free(elem->fields);
9855                 free(elem->actions);
9856                 free(elem->default_action_data);
9857                 free(elem);
9858         }
9859
9860         /* Table types. */
9861         for ( ; ; ) {
9862                 struct table_type *elem;
9863
9864                 elem = TAILQ_FIRST(&p->table_types);
9865                 if (!elem)
9866                         break;
9867
9868                 TAILQ_REMOVE(&p->table_types, elem, node);
9869                 free(elem);
9870         }
9871 }
9872
9873 /*
9874  * Selector.
9875  */
9876 static struct selector *
9877 selector_find(struct rte_swx_pipeline *p, const char *name)
9878 {
9879         struct selector *s;
9880
9881         TAILQ_FOREACH(s, &p->selectors, node)
9882                 if (strcmp(s->name, name) == 0)
9883                         return s;
9884
9885         return NULL;
9886 }
9887
9888 static struct selector *
9889 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9890 {
9891         struct selector *s = NULL;
9892
9893         TAILQ_FOREACH(s, &p->selectors, node)
9894                 if (s->id == id)
9895                         return s;
9896
9897         return NULL;
9898 }
9899
9900 static int
9901 selector_fields_check(struct rte_swx_pipeline *p,
9902                       struct rte_swx_pipeline_selector_params *params,
9903                       struct header **header)
9904 {
9905         struct header *h0 = NULL;
9906         struct field *hf, *mf;
9907         uint32_t i;
9908
9909         /* Return if no selector fields. */
9910         if (!params->n_selector_fields || !params->selector_field_names)
9911                 return -EINVAL;
9912
9913         /* Check that all the selector fields either belong to the same header
9914          * or are all meta-data fields.
9915          */
9916         hf = header_field_parse(p, params->selector_field_names[0], &h0);
9917         mf = metadata_field_parse(p, params->selector_field_names[0]);
9918         if (!hf && !mf)
9919                 return -EINVAL;
9920
9921         for (i = 1; i < params->n_selector_fields; i++)
9922                 if (h0) {
9923                         struct header *h;
9924
9925                         hf = header_field_parse(p, params->selector_field_names[i], &h);
9926                         if (!hf || (h->id != h0->id))
9927                                 return -EINVAL;
9928                 } else {
9929                         mf = metadata_field_parse(p, params->selector_field_names[i]);
9930                         if (!mf)
9931                                 return -EINVAL;
9932                 }
9933
9934         /* Check that there are no duplicated match fields. */
9935         for (i = 0; i < params->n_selector_fields; i++) {
9936                 const char *field_name = params->selector_field_names[i];
9937                 uint32_t j;
9938
9939                 for (j = i + 1; j < params->n_selector_fields; j++)
9940                         if (!strcmp(params->selector_field_names[j], field_name))
9941                                 return -EINVAL;
9942         }
9943
9944         /* Return. */
9945         if (header)
9946                 *header = h0;
9947
9948         return 0;
9949 }
9950
9951 int
9952 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
9953                                  const char *name,
9954                                  struct rte_swx_pipeline_selector_params *params)
9955 {
9956         struct selector *s;
9957         struct header *selector_header = NULL;
9958         struct field *group_id_field, *member_id_field;
9959         uint32_t i;
9960         int status = 0;
9961
9962         CHECK(p, EINVAL);
9963
9964         CHECK_NAME(name, EINVAL);
9965         CHECK(!table_find(p, name), EEXIST);
9966         CHECK(!selector_find(p, name), EEXIST);
9967
9968         CHECK(params, EINVAL);
9969
9970         CHECK_NAME(params->group_id_field_name, EINVAL);
9971         group_id_field = metadata_field_parse(p, params->group_id_field_name);
9972         CHECK(group_id_field, EINVAL);
9973
9974         for (i = 0; i < params->n_selector_fields; i++) {
9975                 const char *field_name = params->selector_field_names[i];
9976
9977                 CHECK_NAME(field_name, EINVAL);
9978         }
9979         status = selector_fields_check(p, params, &selector_header);
9980         if (status)
9981                 return status;
9982
9983         CHECK_NAME(params->member_id_field_name, EINVAL);
9984         member_id_field = metadata_field_parse(p, params->member_id_field_name);
9985         CHECK(member_id_field, EINVAL);
9986
9987         CHECK(params->n_groups_max, EINVAL);
9988
9989         CHECK(params->n_members_per_group_max, EINVAL);
9990
9991         /* Memory allocation. */
9992         s = calloc(1, sizeof(struct selector));
9993         if (!s) {
9994                 status = -ENOMEM;
9995                 goto error;
9996         }
9997
9998         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
9999         if (!s->selector_fields) {
10000                 status = -ENOMEM;
10001                 goto error;
10002         }
10003
10004         /* Node initialization. */
10005         strcpy(s->name, name);
10006
10007         s->group_id_field = group_id_field;
10008
10009         for (i = 0; i < params->n_selector_fields; i++) {
10010                 const char *field_name = params->selector_field_names[i];
10011
10012                 s->selector_fields[i] = selector_header ?
10013                         header_field_parse(p, field_name, NULL) :
10014                         metadata_field_parse(p, field_name);
10015         }
10016
10017         s->n_selector_fields = params->n_selector_fields;
10018
10019         s->selector_header = selector_header;
10020
10021         s->member_id_field = member_id_field;
10022
10023         s->n_groups_max = params->n_groups_max;
10024
10025         s->n_members_per_group_max = params->n_members_per_group_max;
10026
10027         s->id = p->n_selectors;
10028
10029         /* Node add to tailq. */
10030         TAILQ_INSERT_TAIL(&p->selectors, s, node);
10031         p->n_selectors++;
10032
10033         return 0;
10034
10035 error:
10036         if (!s)
10037                 return status;
10038
10039         free(s->selector_fields);
10040
10041         free(s);
10042
10043         return status;
10044 }
10045
10046 static void
10047 selector_params_free(struct rte_swx_table_selector_params *params)
10048 {
10049         if (!params)
10050                 return;
10051
10052         free(params->selector_mask);
10053
10054         free(params);
10055 }
10056
10057 static struct rte_swx_table_selector_params *
10058 selector_table_params_get(struct selector *s)
10059 {
10060         struct rte_swx_table_selector_params *params = NULL;
10061         struct field *first, *last;
10062         uint32_t i;
10063
10064         /* Memory allocation. */
10065         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
10066         if (!params)
10067                 goto error;
10068
10069         /* Group ID. */
10070         params->group_id_offset = s->group_id_field->offset / 8;
10071
10072         /* Find first (smallest offset) and last (biggest offset) selector fields. */
10073         first = s->selector_fields[0];
10074         last = s->selector_fields[0];
10075
10076         for (i = 0; i < s->n_selector_fields; i++) {
10077                 struct field *f = s->selector_fields[i];
10078
10079                 if (f->offset < first->offset)
10080                         first = f;
10081
10082                 if (f->offset > last->offset)
10083                         last = f;
10084         }
10085
10086         /* Selector offset and size. */
10087         params->selector_offset = first->offset / 8;
10088         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
10089
10090         /* Memory allocation. */
10091         params->selector_mask = calloc(1, params->selector_size);
10092         if (!params->selector_mask)
10093                 goto error;
10094
10095         /* Selector mask. */
10096         for (i = 0; i < s->n_selector_fields; i++) {
10097                 struct field *f = s->selector_fields[i];
10098                 uint32_t start = (f->offset - first->offset) / 8;
10099                 size_t size = f->n_bits / 8;
10100
10101                 memset(&params->selector_mask[start], 0xFF, size);
10102         }
10103
10104         /* Member ID. */
10105         params->member_id_offset = s->member_id_field->offset / 8;
10106
10107         /* Maximum number of groups. */
10108         params->n_groups_max = s->n_groups_max;
10109
10110         /* Maximum number of members per group. */
10111         params->n_members_per_group_max = s->n_members_per_group_max;
10112
10113         return params;
10114
10115 error:
10116         selector_params_free(params);
10117         return NULL;
10118 }
10119
10120 static void
10121 selector_build_free(struct rte_swx_pipeline *p)
10122 {
10123         uint32_t i;
10124
10125         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10126                 struct thread *t = &p->threads[i];
10127                 uint32_t j;
10128
10129                 if (!t->selectors)
10130                         continue;
10131
10132                 for (j = 0; j < p->n_selectors; j++) {
10133                         struct selector_runtime *r = &t->selectors[j];
10134
10135                         free(r->mailbox);
10136                 }
10137
10138                 free(t->selectors);
10139                 t->selectors = NULL;
10140         }
10141
10142         free(p->selector_stats);
10143         p->selector_stats = NULL;
10144 }
10145
10146 static int
10147 selector_build(struct rte_swx_pipeline *p)
10148 {
10149         uint32_t i;
10150         int status = 0;
10151
10152         /* Per pipeline: selector statistics. */
10153         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
10154         if (!p->selector_stats) {
10155                 status = -ENOMEM;
10156                 goto error;
10157         }
10158
10159         /* Per thread: selector run-time. */
10160         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10161                 struct thread *t = &p->threads[i];
10162                 struct selector *s;
10163
10164                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
10165                 if (!t->selectors) {
10166                         status = -ENOMEM;
10167                         goto error;
10168                 }
10169
10170                 TAILQ_FOREACH(s, &p->selectors, node) {
10171                         struct selector_runtime *r = &t->selectors[s->id];
10172                         uint64_t size;
10173
10174                         /* r->mailbox. */
10175                         size = rte_swx_table_selector_mailbox_size_get();
10176                         if (size) {
10177                                 r->mailbox = calloc(1, size);
10178                                 if (!r->mailbox) {
10179                                         status = -ENOMEM;
10180                                         goto error;
10181                                 }
10182                         }
10183
10184                         /* r->group_id_buffer. */
10185                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
10186
10187                         /* r->selector_buffer. */
10188                         r->selector_buffer = s->selector_header ?
10189                                 &t->structs[s->selector_header->struct_id] :
10190                                 &t->structs[p->metadata_struct_id];
10191
10192                         /* r->member_id_buffer. */
10193                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
10194                 }
10195         }
10196
10197         return 0;
10198
10199 error:
10200         selector_build_free(p);
10201         return status;
10202 }
10203
10204 static void
10205 selector_free(struct rte_swx_pipeline *p)
10206 {
10207         selector_build_free(p);
10208
10209         /* Selector tables. */
10210         for ( ; ; ) {
10211                 struct selector *elem;
10212
10213                 elem = TAILQ_FIRST(&p->selectors);
10214                 if (!elem)
10215                         break;
10216
10217                 TAILQ_REMOVE(&p->selectors, elem, node);
10218                 free(elem->selector_fields);
10219                 free(elem);
10220         }
10221 }
10222
10223 /*
10224  * Table state.
10225  */
10226 static int
10227 table_state_build(struct rte_swx_pipeline *p)
10228 {
10229         struct table *table;
10230         struct selector *s;
10231
10232         p->table_state = calloc(p->n_tables + p->n_selectors,
10233                                 sizeof(struct rte_swx_table_state));
10234         CHECK(p->table_state, ENOMEM);
10235
10236         TAILQ_FOREACH(table, &p->tables, node) {
10237                 struct rte_swx_table_state *ts = &p->table_state[table->id];
10238
10239                 if (table->type) {
10240                         struct rte_swx_table_params *params;
10241
10242                         /* ts->obj. */
10243                         params = table_params_get(table);
10244                         CHECK(params, ENOMEM);
10245
10246                         ts->obj = table->type->ops.create(params,
10247                                 NULL,
10248                                 table->args,
10249                                 p->numa_node);
10250
10251                         table_params_free(params);
10252                         CHECK(ts->obj, ENODEV);
10253                 }
10254
10255                 /* ts->default_action_data. */
10256                 if (table->action_data_size_max) {
10257                         ts->default_action_data =
10258                                 malloc(table->action_data_size_max);
10259                         CHECK(ts->default_action_data, ENOMEM);
10260
10261                         memcpy(ts->default_action_data,
10262                                table->default_action_data,
10263                                table->action_data_size_max);
10264                 }
10265
10266                 /* ts->default_action_id. */
10267                 ts->default_action_id = table->default_action->id;
10268         }
10269
10270         TAILQ_FOREACH(s, &p->selectors, node) {
10271                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
10272                 struct rte_swx_table_selector_params *params;
10273
10274                 /* ts->obj. */
10275                 params = selector_table_params_get(s);
10276                 CHECK(params, ENOMEM);
10277
10278                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
10279
10280                 selector_params_free(params);
10281                 CHECK(ts->obj, ENODEV);
10282         }
10283
10284         return 0;
10285 }
10286
10287 static void
10288 table_state_build_free(struct rte_swx_pipeline *p)
10289 {
10290         uint32_t i;
10291
10292         if (!p->table_state)
10293                 return;
10294
10295         for (i = 0; i < p->n_tables; i++) {
10296                 struct rte_swx_table_state *ts = &p->table_state[i];
10297                 struct table *table = table_find_by_id(p, i);
10298
10299                 /* ts->obj. */
10300                 if (table->type && ts->obj)
10301                         table->type->ops.free(ts->obj);
10302
10303                 /* ts->default_action_data. */
10304                 free(ts->default_action_data);
10305         }
10306
10307         for (i = 0; i < p->n_selectors; i++) {
10308                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
10309
10310                 /* ts->obj. */
10311                 if (ts->obj)
10312                         rte_swx_table_selector_free(ts->obj);
10313         }
10314
10315         free(p->table_state);
10316         p->table_state = NULL;
10317 }
10318
10319 static void
10320 table_state_free(struct rte_swx_pipeline *p)
10321 {
10322         table_state_build_free(p);
10323 }
10324
10325 /*
10326  * Register array.
10327  */
10328 static struct regarray *
10329 regarray_find(struct rte_swx_pipeline *p, const char *name)
10330 {
10331         struct regarray *elem;
10332
10333         TAILQ_FOREACH(elem, &p->regarrays, node)
10334                 if (!strcmp(elem->name, name))
10335                         return elem;
10336
10337         return NULL;
10338 }
10339
10340 static struct regarray *
10341 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
10342 {
10343         struct regarray *elem = NULL;
10344
10345         TAILQ_FOREACH(elem, &p->regarrays, node)
10346                 if (elem->id == id)
10347                         return elem;
10348
10349         return NULL;
10350 }
10351
10352 int
10353 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
10354                               const char *name,
10355                               uint32_t size,
10356                               uint64_t init_val)
10357 {
10358         struct regarray *r;
10359
10360         CHECK(p, EINVAL);
10361
10362         CHECK_NAME(name, EINVAL);
10363         CHECK(!regarray_find(p, name), EEXIST);
10364
10365         CHECK(size, EINVAL);
10366         size = rte_align32pow2(size);
10367
10368         /* Memory allocation. */
10369         r = calloc(1, sizeof(struct regarray));
10370         CHECK(r, ENOMEM);
10371
10372         /* Node initialization. */
10373         strcpy(r->name, name);
10374         r->init_val = init_val;
10375         r->size = size;
10376         r->id = p->n_regarrays;
10377
10378         /* Node add to tailq. */
10379         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
10380         p->n_regarrays++;
10381
10382         return 0;
10383 }
10384
10385 static int
10386 regarray_build(struct rte_swx_pipeline *p)
10387 {
10388         struct regarray *regarray;
10389
10390         if (!p->n_regarrays)
10391                 return 0;
10392
10393         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
10394         CHECK(p->regarray_runtime, ENOMEM);
10395
10396         TAILQ_FOREACH(regarray, &p->regarrays, node) {
10397                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
10398                 uint32_t i;
10399
10400                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
10401                                          RTE_CACHE_LINE_SIZE,
10402                                          p->numa_node);
10403                 CHECK(r->regarray, ENOMEM);
10404
10405                 if (regarray->init_val)
10406                         for (i = 0; i < regarray->size; i++)
10407                                 r->regarray[i] = regarray->init_val;
10408
10409                 r->size_mask = regarray->size - 1;
10410         }
10411
10412         return 0;
10413 }
10414
10415 static void
10416 regarray_build_free(struct rte_swx_pipeline *p)
10417 {
10418         uint32_t i;
10419
10420         if (!p->regarray_runtime)
10421                 return;
10422
10423         for (i = 0; i < p->n_regarrays; i++) {
10424                 struct regarray *regarray = regarray_find_by_id(p, i);
10425                 struct regarray_runtime *r = &p->regarray_runtime[i];
10426
10427                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
10428         }
10429
10430         free(p->regarray_runtime);
10431         p->regarray_runtime = NULL;
10432 }
10433
10434 static void
10435 regarray_free(struct rte_swx_pipeline *p)
10436 {
10437         regarray_build_free(p);
10438
10439         for ( ; ; ) {
10440                 struct regarray *elem;
10441
10442                 elem = TAILQ_FIRST(&p->regarrays);
10443                 if (!elem)
10444                         break;
10445
10446                 TAILQ_REMOVE(&p->regarrays, elem, node);
10447                 free(elem);
10448         }
10449 }
10450
10451 /*
10452  * Meter array.
10453  */
10454 static struct meter_profile *
10455 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
10456 {
10457         struct meter_profile *elem;
10458
10459         TAILQ_FOREACH(elem, &p->meter_profiles, node)
10460                 if (!strcmp(elem->name, name))
10461                         return elem;
10462
10463         return NULL;
10464 }
10465
10466 static struct metarray *
10467 metarray_find(struct rte_swx_pipeline *p, const char *name)
10468 {
10469         struct metarray *elem;
10470
10471         TAILQ_FOREACH(elem, &p->metarrays, node)
10472                 if (!strcmp(elem->name, name))
10473                         return elem;
10474
10475         return NULL;
10476 }
10477
10478 static struct metarray *
10479 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
10480 {
10481         struct metarray *elem = NULL;
10482
10483         TAILQ_FOREACH(elem, &p->metarrays, node)
10484                 if (elem->id == id)
10485                         return elem;
10486
10487         return NULL;
10488 }
10489
10490 int
10491 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
10492                                  const char *name,
10493                                  uint32_t size)
10494 {
10495         struct metarray *m;
10496
10497         CHECK(p, EINVAL);
10498
10499         CHECK_NAME(name, EINVAL);
10500         CHECK(!metarray_find(p, name), EEXIST);
10501
10502         CHECK(size, EINVAL);
10503         size = rte_align32pow2(size);
10504
10505         /* Memory allocation. */
10506         m = calloc(1, sizeof(struct metarray));
10507         CHECK(m, ENOMEM);
10508
10509         /* Node initialization. */
10510         strcpy(m->name, name);
10511         m->size = size;
10512         m->id = p->n_metarrays;
10513
10514         /* Node add to tailq. */
10515         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
10516         p->n_metarrays++;
10517
10518         return 0;
10519 }
10520
10521 struct meter_profile meter_profile_default = {
10522         .node = {0},
10523         .name = "",
10524         .params = {0},
10525
10526         .profile = {
10527                 .cbs = 10000,
10528                 .pbs = 10000,
10529                 .cir_period = 1,
10530                 .cir_bytes_per_period = 1,
10531                 .pir_period = 1,
10532                 .pir_bytes_per_period = 1,
10533         },
10534
10535         .n_users = 0,
10536 };
10537
10538 static void
10539 meter_init(struct meter *m)
10540 {
10541         memset(m, 0, sizeof(struct meter));
10542         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
10543         m->profile = &meter_profile_default;
10544         m->color_mask = RTE_COLOR_GREEN;
10545
10546         meter_profile_default.n_users++;
10547 }
10548
10549 static int
10550 metarray_build(struct rte_swx_pipeline *p)
10551 {
10552         struct metarray *m;
10553
10554         if (!p->n_metarrays)
10555                 return 0;
10556
10557         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
10558         CHECK(p->metarray_runtime, ENOMEM);
10559
10560         TAILQ_FOREACH(m, &p->metarrays, node) {
10561                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
10562                 uint32_t i;
10563
10564                 r->metarray = env_malloc(m->size * sizeof(struct meter),
10565                                          RTE_CACHE_LINE_SIZE,
10566                                          p->numa_node);
10567                 CHECK(r->metarray, ENOMEM);
10568
10569                 for (i = 0; i < m->size; i++)
10570                         meter_init(&r->metarray[i]);
10571
10572                 r->size_mask = m->size - 1;
10573         }
10574
10575         return 0;
10576 }
10577
10578 static void
10579 metarray_build_free(struct rte_swx_pipeline *p)
10580 {
10581         uint32_t i;
10582
10583         if (!p->metarray_runtime)
10584                 return;
10585
10586         for (i = 0; i < p->n_metarrays; i++) {
10587                 struct metarray *m = metarray_find_by_id(p, i);
10588                 struct metarray_runtime *r = &p->metarray_runtime[i];
10589
10590                 env_free(r->metarray, m->size * sizeof(struct meter));
10591         }
10592
10593         free(p->metarray_runtime);
10594         p->metarray_runtime = NULL;
10595 }
10596
10597 static void
10598 metarray_free(struct rte_swx_pipeline *p)
10599 {
10600         metarray_build_free(p);
10601
10602         /* Meter arrays. */
10603         for ( ; ; ) {
10604                 struct metarray *elem;
10605
10606                 elem = TAILQ_FIRST(&p->metarrays);
10607                 if (!elem)
10608                         break;
10609
10610                 TAILQ_REMOVE(&p->metarrays, elem, node);
10611                 free(elem);
10612         }
10613
10614         /* Meter profiles. */
10615         for ( ; ; ) {
10616                 struct meter_profile *elem;
10617
10618                 elem = TAILQ_FIRST(&p->meter_profiles);
10619                 if (!elem)
10620                         break;
10621
10622                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
10623                 free(elem);
10624         }
10625 }
10626
10627 /*
10628  * Pipeline.
10629  */
10630 int
10631 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
10632 {
10633         struct rte_swx_pipeline *pipeline;
10634
10635         /* Check input parameters. */
10636         CHECK(p, EINVAL);
10637
10638         /* Memory allocation. */
10639         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
10640         CHECK(pipeline, ENOMEM);
10641
10642         /* Initialization. */
10643         TAILQ_INIT(&pipeline->struct_types);
10644         TAILQ_INIT(&pipeline->port_in_types);
10645         TAILQ_INIT(&pipeline->ports_in);
10646         TAILQ_INIT(&pipeline->port_out_types);
10647         TAILQ_INIT(&pipeline->ports_out);
10648         TAILQ_INIT(&pipeline->extern_types);
10649         TAILQ_INIT(&pipeline->extern_objs);
10650         TAILQ_INIT(&pipeline->extern_funcs);
10651         TAILQ_INIT(&pipeline->headers);
10652         TAILQ_INIT(&pipeline->actions);
10653         TAILQ_INIT(&pipeline->table_types);
10654         TAILQ_INIT(&pipeline->tables);
10655         TAILQ_INIT(&pipeline->selectors);
10656         TAILQ_INIT(&pipeline->regarrays);
10657         TAILQ_INIT(&pipeline->meter_profiles);
10658         TAILQ_INIT(&pipeline->metarrays);
10659
10660         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
10661         pipeline->numa_node = numa_node;
10662
10663         *p = pipeline;
10664         return 0;
10665 }
10666
10667 void
10668 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
10669 {
10670         if (!p)
10671                 return;
10672
10673         free(p->instructions);
10674
10675         metarray_free(p);
10676         regarray_free(p);
10677         table_state_free(p);
10678         selector_free(p);
10679         table_free(p);
10680         action_free(p);
10681         metadata_free(p);
10682         header_free(p);
10683         extern_func_free(p);
10684         extern_obj_free(p);
10685         port_out_free(p);
10686         port_in_free(p);
10687         struct_free(p);
10688
10689         free(p);
10690 }
10691
10692 int
10693 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
10694                                      const char **instructions,
10695                                      uint32_t n_instructions)
10696 {
10697         int err;
10698         uint32_t i;
10699
10700         err = instruction_config(p, NULL, instructions, n_instructions);
10701         if (err)
10702                 return err;
10703
10704         /* Thread instruction pointer reset. */
10705         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10706                 struct thread *t = &p->threads[i];
10707
10708                 thread_ip_reset(p, t);
10709         }
10710
10711         return 0;
10712 }
10713
10714 int
10715 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
10716 {
10717         int status;
10718
10719         CHECK(p, EINVAL);
10720         CHECK(p->build_done == 0, EEXIST);
10721
10722         status = port_in_build(p);
10723         if (status)
10724                 goto error;
10725
10726         status = port_out_build(p);
10727         if (status)
10728                 goto error;
10729
10730         status = struct_build(p);
10731         if (status)
10732                 goto error;
10733
10734         status = extern_obj_build(p);
10735         if (status)
10736                 goto error;
10737
10738         status = extern_func_build(p);
10739         if (status)
10740                 goto error;
10741
10742         status = header_build(p);
10743         if (status)
10744                 goto error;
10745
10746         status = metadata_build(p);
10747         if (status)
10748                 goto error;
10749
10750         status = action_build(p);
10751         if (status)
10752                 goto error;
10753
10754         status = table_build(p);
10755         if (status)
10756                 goto error;
10757
10758         status = selector_build(p);
10759         if (status)
10760                 goto error;
10761
10762         status = table_state_build(p);
10763         if (status)
10764                 goto error;
10765
10766         status = regarray_build(p);
10767         if (status)
10768                 goto error;
10769
10770         status = metarray_build(p);
10771         if (status)
10772                 goto error;
10773
10774         p->build_done = 1;
10775         return 0;
10776
10777 error:
10778         metarray_build_free(p);
10779         regarray_build_free(p);
10780         table_state_build_free(p);
10781         selector_build_free(p);
10782         table_build_free(p);
10783         action_build_free(p);
10784         metadata_build_free(p);
10785         header_build_free(p);
10786         extern_func_build_free(p);
10787         extern_obj_build_free(p);
10788         port_out_build_free(p);
10789         port_in_build_free(p);
10790         struct_build_free(p);
10791
10792         return status;
10793 }
10794
10795 void
10796 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
10797 {
10798         uint32_t i;
10799
10800         for (i = 0; i < n_instructions; i++)
10801                 instr_exec(p);
10802 }
10803
10804 void
10805 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
10806 {
10807         uint32_t i;
10808
10809         for (i = 0; i < p->n_ports_out; i++) {
10810                 struct port_out_runtime *port = &p->out[i];
10811
10812                 if (port->flush)
10813                         port->flush(port->obj);
10814         }
10815 }
10816
10817 /*
10818  * Control.
10819  */
10820 int
10821 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
10822                               struct rte_swx_ctl_pipeline_info *pipeline)
10823 {
10824         struct action *action;
10825         struct table *table;
10826         uint32_t n_actions = 0, n_tables = 0;
10827
10828         if (!p || !pipeline)
10829                 return -EINVAL;
10830
10831         TAILQ_FOREACH(action, &p->actions, node)
10832                 n_actions++;
10833
10834         TAILQ_FOREACH(table, &p->tables, node)
10835                 n_tables++;
10836
10837         pipeline->n_ports_in = p->n_ports_in;
10838         pipeline->n_ports_out = p->n_ports_out;
10839         pipeline->n_actions = n_actions;
10840         pipeline->n_tables = n_tables;
10841         pipeline->n_selectors = p->n_selectors;
10842         pipeline->n_regarrays = p->n_regarrays;
10843         pipeline->n_metarrays = p->n_metarrays;
10844
10845         return 0;
10846 }
10847
10848 int
10849 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
10850 {
10851         if (!p || !numa_node)
10852                 return -EINVAL;
10853
10854         *numa_node = p->numa_node;
10855         return 0;
10856 }
10857
10858 int
10859 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
10860                             uint32_t action_id,
10861                             struct rte_swx_ctl_action_info *action)
10862 {
10863         struct action *a = NULL;
10864
10865         if (!p || (action_id >= p->n_actions) || !action)
10866                 return -EINVAL;
10867
10868         a = action_find_by_id(p, action_id);
10869         if (!a)
10870                 return -EINVAL;
10871
10872         strcpy(action->name, a->name);
10873         action->n_args = a->st ? a->st->n_fields : 0;
10874         return 0;
10875 }
10876
10877 int
10878 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10879                                 uint32_t action_id,
10880                                 uint32_t action_arg_id,
10881                                 struct rte_swx_ctl_action_arg_info *action_arg)
10882 {
10883         struct action *a = NULL;
10884         struct field *arg = NULL;
10885
10886         if (!p || (action_id >= p->n_actions) || !action_arg)
10887                 return -EINVAL;
10888
10889         a = action_find_by_id(p, action_id);
10890         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10891                 return -EINVAL;
10892
10893         arg = &a->st->fields[action_arg_id];
10894         strcpy(action_arg->name, arg->name);
10895         action_arg->n_bits = arg->n_bits;
10896         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10897
10898         return 0;
10899 }
10900
10901 int
10902 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10903                            uint32_t table_id,
10904                            struct rte_swx_ctl_table_info *table)
10905 {
10906         struct table *t = NULL;
10907
10908         if (!p || !table)
10909                 return -EINVAL;
10910
10911         t = table_find_by_id(p, table_id);
10912         if (!t)
10913                 return -EINVAL;
10914
10915         strcpy(table->name, t->name);
10916         strcpy(table->args, t->args);
10917         table->n_match_fields = t->n_fields;
10918         table->n_actions = t->n_actions;
10919         table->default_action_is_const = t->default_action_is_const;
10920         table->size = t->size;
10921         return 0;
10922 }
10923
10924 int
10925 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10926         uint32_t table_id,
10927         uint32_t match_field_id,
10928         struct rte_swx_ctl_table_match_field_info *match_field)
10929 {
10930         struct table *t;
10931         struct match_field *f;
10932
10933         if (!p || (table_id >= p->n_tables) || !match_field)
10934                 return -EINVAL;
10935
10936         t = table_find_by_id(p, table_id);
10937         if (!t || (match_field_id >= t->n_fields))
10938                 return -EINVAL;
10939
10940         f = &t->fields[match_field_id];
10941         match_field->match_type = f->match_type;
10942         match_field->is_header = t->header ? 1 : 0;
10943         match_field->n_bits = f->field->n_bits;
10944         match_field->offset = f->field->offset;
10945
10946         return 0;
10947 }
10948
10949 int
10950 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10951         uint32_t table_id,
10952         uint32_t table_action_id,
10953         struct rte_swx_ctl_table_action_info *table_action)
10954 {
10955         struct table *t;
10956
10957         if (!p || (table_id >= p->n_tables) || !table_action)
10958                 return -EINVAL;
10959
10960         t = table_find_by_id(p, table_id);
10961         if (!t || (table_action_id >= t->n_actions))
10962                 return -EINVAL;
10963
10964         table_action->action_id = t->actions[table_action_id]->id;
10965
10966         return 0;
10967 }
10968
10969 int
10970 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10971                           uint32_t table_id,
10972                           struct rte_swx_table_ops *table_ops,
10973                           int *is_stub)
10974 {
10975         struct table *t;
10976
10977         if (!p || (table_id >= p->n_tables))
10978                 return -EINVAL;
10979
10980         t = table_find_by_id(p, table_id);
10981         if (!t)
10982                 return -EINVAL;
10983
10984         if (t->type) {
10985                 if (table_ops)
10986                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10987                 *is_stub = 0;
10988         } else {
10989                 *is_stub = 1;
10990         }
10991
10992         return 0;
10993 }
10994
10995 int
10996 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10997                               uint32_t selector_id,
10998                               struct rte_swx_ctl_selector_info *selector)
10999 {
11000         struct selector *s = NULL;
11001
11002         if (!p || !selector)
11003                 return -EINVAL;
11004
11005         s = selector_find_by_id(p, selector_id);
11006         if (!s)
11007                 return -EINVAL;
11008
11009         strcpy(selector->name, s->name);
11010
11011         selector->n_selector_fields = s->n_selector_fields;
11012         selector->n_groups_max = s->n_groups_max;
11013         selector->n_members_per_group_max = s->n_members_per_group_max;
11014
11015         return 0;
11016 }
11017
11018 int
11019 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
11020          uint32_t selector_id,
11021          struct rte_swx_ctl_table_match_field_info *field)
11022 {
11023         struct selector *s;
11024
11025         if (!p || (selector_id >= p->n_selectors) || !field)
11026                 return -EINVAL;
11027
11028         s = selector_find_by_id(p, selector_id);
11029         if (!s)
11030                 return -EINVAL;
11031
11032         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
11033         field->is_header = 0;
11034         field->n_bits = s->group_id_field->n_bits;
11035         field->offset = s->group_id_field->offset;
11036
11037         return 0;
11038 }
11039
11040 int
11041 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
11042          uint32_t selector_id,
11043          uint32_t selector_field_id,
11044          struct rte_swx_ctl_table_match_field_info *field)
11045 {
11046         struct selector *s;
11047         struct field *f;
11048
11049         if (!p || (selector_id >= p->n_selectors) || !field)
11050                 return -EINVAL;
11051
11052         s = selector_find_by_id(p, selector_id);
11053         if (!s || (selector_field_id >= s->n_selector_fields))
11054                 return -EINVAL;
11055
11056         f = s->selector_fields[selector_field_id];
11057         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
11058         field->is_header = s->selector_header ? 1 : 0;
11059         field->n_bits = f->n_bits;
11060         field->offset = f->offset;
11061
11062         return 0;
11063 }
11064
11065 int
11066 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
11067          uint32_t selector_id,
11068          struct rte_swx_ctl_table_match_field_info *field)
11069 {
11070         struct selector *s;
11071
11072         if (!p || (selector_id >= p->n_selectors) || !field)
11073                 return -EINVAL;
11074
11075         s = selector_find_by_id(p, selector_id);
11076         if (!s)
11077                 return -EINVAL;
11078
11079         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
11080         field->is_header = 0;
11081         field->n_bits = s->member_id_field->n_bits;
11082         field->offset = s->member_id_field->offset;
11083
11084         return 0;
11085 }
11086
11087 int
11088 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
11089                                  struct rte_swx_table_state **table_state)
11090 {
11091         if (!p || !table_state || !p->build_done)
11092                 return -EINVAL;
11093
11094         *table_state = p->table_state;
11095         return 0;
11096 }
11097
11098 int
11099 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
11100                                  struct rte_swx_table_state *table_state)
11101 {
11102         if (!p || !table_state || !p->build_done)
11103                 return -EINVAL;
11104
11105         p->table_state = table_state;
11106         return 0;
11107 }
11108
11109 int
11110 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
11111                                         uint32_t port_id,
11112                                         struct rte_swx_port_in_stats *stats)
11113 {
11114         struct port_in *port;
11115
11116         if (!p || !stats)
11117                 return -EINVAL;
11118
11119         port = port_in_find(p, port_id);
11120         if (!port)
11121                 return -EINVAL;
11122
11123         port->type->ops.stats_read(port->obj, stats);
11124         return 0;
11125 }
11126
11127 int
11128 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
11129                                          uint32_t port_id,
11130                                          struct rte_swx_port_out_stats *stats)
11131 {
11132         struct port_out *port;
11133
11134         if (!p || !stats)
11135                 return -EINVAL;
11136
11137         port = port_out_find(p, port_id);
11138         if (!port)
11139                 return -EINVAL;
11140
11141         port->type->ops.stats_read(port->obj, stats);
11142         return 0;
11143 }
11144
11145 int
11146 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
11147                                       const char *table_name,
11148                                       struct rte_swx_table_stats *stats)
11149 {
11150         struct table *table;
11151         struct table_statistics *table_stats;
11152
11153         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
11154                 return -EINVAL;
11155
11156         table = table_find(p, table_name);
11157         if (!table)
11158                 return -EINVAL;
11159
11160         table_stats = &p->table_stats[table->id];
11161
11162         memcpy(stats->n_pkts_action,
11163                table_stats->n_pkts_action,
11164                p->n_actions * sizeof(uint64_t));
11165
11166         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
11167         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
11168
11169         return 0;
11170 }
11171
11172 int
11173 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
11174         const char *selector_name,
11175         struct rte_swx_pipeline_selector_stats *stats)
11176 {
11177         struct selector *s;
11178
11179         if (!p || !selector_name || !selector_name[0] || !stats)
11180                 return -EINVAL;
11181
11182         s = selector_find(p, selector_name);
11183         if (!s)
11184                 return -EINVAL;
11185
11186         stats->n_pkts = p->selector_stats[s->id].n_pkts;
11187
11188         return 0;
11189 }
11190
11191 int
11192 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
11193                               uint32_t regarray_id,
11194                               struct rte_swx_ctl_regarray_info *regarray)
11195 {
11196         struct regarray *r;
11197
11198         if (!p || !regarray)
11199                 return -EINVAL;
11200
11201         r = regarray_find_by_id(p, regarray_id);
11202         if (!r)
11203                 return -EINVAL;
11204
11205         strcpy(regarray->name, r->name);
11206         regarray->size = r->size;
11207         return 0;
11208 }
11209
11210 int
11211 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
11212                                    const char *regarray_name,
11213                                    uint32_t regarray_index,
11214                                    uint64_t *value)
11215 {
11216         struct regarray *regarray;
11217         struct regarray_runtime *r;
11218
11219         if (!p || !regarray_name || !value)
11220                 return -EINVAL;
11221
11222         regarray = regarray_find(p, regarray_name);
11223         if (!regarray || (regarray_index >= regarray->size))
11224                 return -EINVAL;
11225
11226         r = &p->regarray_runtime[regarray->id];
11227         *value = r->regarray[regarray_index];
11228         return 0;
11229 }
11230
11231 int
11232 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
11233                                    const char *regarray_name,
11234                                    uint32_t regarray_index,
11235                                    uint64_t value)
11236 {
11237         struct regarray *regarray;
11238         struct regarray_runtime *r;
11239
11240         if (!p || !regarray_name)
11241                 return -EINVAL;
11242
11243         regarray = regarray_find(p, regarray_name);
11244         if (!regarray || (regarray_index >= regarray->size))
11245                 return -EINVAL;
11246
11247         r = &p->regarray_runtime[regarray->id];
11248         r->regarray[regarray_index] = value;
11249         return 0;
11250 }
11251
11252 int
11253 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
11254                               uint32_t metarray_id,
11255                               struct rte_swx_ctl_metarray_info *metarray)
11256 {
11257         struct metarray *m;
11258
11259         if (!p || !metarray)
11260                 return -EINVAL;
11261
11262         m = metarray_find_by_id(p, metarray_id);
11263         if (!m)
11264                 return -EINVAL;
11265
11266         strcpy(metarray->name, m->name);
11267         metarray->size = m->size;
11268         return 0;
11269 }
11270
11271 int
11272 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
11273                               const char *name,
11274                               struct rte_meter_trtcm_params *params)
11275 {
11276         struct meter_profile *mp;
11277         int status;
11278
11279         CHECK(p, EINVAL);
11280         CHECK_NAME(name, EINVAL);
11281         CHECK(params, EINVAL);
11282         CHECK(!meter_profile_find(p, name), EEXIST);
11283
11284         /* Node allocation. */
11285         mp = calloc(1, sizeof(struct meter_profile));
11286         CHECK(mp, ENOMEM);
11287
11288         /* Node initialization. */
11289         strcpy(mp->name, name);
11290         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
11291         status = rte_meter_trtcm_profile_config(&mp->profile, params);
11292         if (status) {
11293                 free(mp);
11294                 CHECK(0, EINVAL);
11295         }
11296
11297         /* Node add to tailq. */
11298         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
11299
11300         return 0;
11301 }
11302
11303 int
11304 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
11305                                  const char *name)
11306 {
11307         struct meter_profile *mp;
11308
11309         CHECK(p, EINVAL);
11310         CHECK_NAME(name, EINVAL);
11311
11312         mp = meter_profile_find(p, name);
11313         CHECK(mp, EINVAL);
11314         CHECK(!mp->n_users, EBUSY);
11315
11316         /* Remove node from tailq. */
11317         TAILQ_REMOVE(&p->meter_profiles, mp, node);
11318         free(mp);
11319
11320         return 0;
11321 }
11322
11323 int
11324 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
11325                         const char *metarray_name,
11326                         uint32_t metarray_index)
11327 {
11328         struct meter_profile *mp_old;
11329         struct metarray *metarray;
11330         struct metarray_runtime *metarray_runtime;
11331         struct meter *m;
11332
11333         CHECK(p, EINVAL);
11334         CHECK_NAME(metarray_name, EINVAL);
11335
11336         metarray = metarray_find(p, metarray_name);
11337         CHECK(metarray, EINVAL);
11338         CHECK(metarray_index < metarray->size, EINVAL);
11339
11340         metarray_runtime = &p->metarray_runtime[metarray->id];
11341         m = &metarray_runtime->metarray[metarray_index];
11342         mp_old = m->profile;
11343
11344         meter_init(m);
11345
11346         mp_old->n_users--;
11347
11348         return 0;
11349 }
11350
11351 int
11352 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
11353                       const char *metarray_name,
11354                       uint32_t metarray_index,
11355                       const char *profile_name)
11356 {
11357         struct meter_profile *mp, *mp_old;
11358         struct metarray *metarray;
11359         struct metarray_runtime *metarray_runtime;
11360         struct meter *m;
11361
11362         CHECK(p, EINVAL);
11363         CHECK_NAME(metarray_name, EINVAL);
11364
11365         metarray = metarray_find(p, metarray_name);
11366         CHECK(metarray, EINVAL);
11367         CHECK(metarray_index < metarray->size, EINVAL);
11368
11369         mp = meter_profile_find(p, profile_name);
11370         CHECK(mp, EINVAL);
11371
11372         metarray_runtime = &p->metarray_runtime[metarray->id];
11373         m = &metarray_runtime->metarray[metarray_index];
11374         mp_old = m->profile;
11375
11376         memset(m, 0, sizeof(struct meter));
11377         rte_meter_trtcm_config(&m->m, &mp->profile);
11378         m->profile = mp;
11379         m->color_mask = RTE_COLORS;
11380
11381         mp->n_users++;
11382         mp_old->n_users--;
11383
11384         return 0;
11385 }
11386
11387 int
11388 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
11389                              const char *metarray_name,
11390                              uint32_t metarray_index,
11391                              struct rte_swx_ctl_meter_stats *stats)
11392 {
11393         struct metarray *metarray;
11394         struct metarray_runtime *metarray_runtime;
11395         struct meter *m;
11396
11397         CHECK(p, EINVAL);
11398         CHECK_NAME(metarray_name, EINVAL);
11399
11400         metarray = metarray_find(p, metarray_name);
11401         CHECK(metarray, EINVAL);
11402         CHECK(metarray_index < metarray->size, EINVAL);
11403
11404         CHECK(stats, EINVAL);
11405
11406         metarray_runtime = &p->metarray_runtime[metarray->id];
11407         m = &metarray_runtime->metarray[metarray_index];
11408
11409         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
11410         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
11411
11412         return 0;
11413 }