lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <inttypes.h>
   9 #include <sys/queue.h>
  10 #include <arpa/inet.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_prefetch.h>
  14 #include <rte_byteorder.h>
  15 #include <rte_cycles.h>
  16 #include <rte_meter.h>
  17
  18 #include "rte_swx_pipeline.h"
  19 #include "rte_swx_ctl.h"
  20
  21 #define CHECK(condition, err_code)                                             \
  22 do {                                                                           \
  23         if (!(condition))                                                      \
  24                 return -(err_code);                                            \
  25 } while (0)
  26
  27 #define CHECK_NAME(name, err_code)                                             \
  28         CHECK((name) &&                                                        \
  29               (name)[0] &&                                                     \
  30               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  31               err_code)
  32
  33 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  34         CHECK((instr) &&                                                       \
  35               (instr)[0] &&                                                    \
  36               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  37                RTE_SWX_INSTRUCTION_SIZE),                                      \
  38               err_code)
  39
  40 #ifndef TRACE_LEVEL
  41 #define TRACE_LEVEL 0
  42 #endif
  43
  44 #if TRACE_LEVEL
  45 #define TRACE(...) printf(__VA_ARGS__)
  46 #else
  47 #define TRACE(...)
  48 #endif
  49
  50 /*
  51  * Environment.
  52  */
  53 #define ntoh64(x) rte_be_to_cpu_64(x)
  54 #define hton64(x) rte_cpu_to_be_64(x)
  55
  56 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  57
  58 #include <rte_malloc.h>
  59
  60 static void *
  61 env_malloc(size_t size, size_t alignment, int numa_node)
  62 {
  63         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  64 }
  65
  66 static void
  67 env_free(void *start, size_t size __rte_unused)
  68 {
  69         rte_free(start);
  70 }
  71
  72 #else
  73
  74 #include <numa.h>
  75
  76 static void *
  77 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  78 {
  79         void *start;
  80
  81         if (numa_available() == -1)
  82                 return NULL;
  83
  84         start = numa_alloc_onnode(size, numa_node);
  85         if (!start)
  86                 return NULL;
  87
  88         memset(start, 0, size);
  89         return start;
  90 }
  91
  92 static void
  93 env_free(void *start, size_t size)
  94 {
  95         if (numa_available() == -1)
  96                 return;
  97
  98         numa_free(start, size);
  99 }
 100
 101 #endif
 102
 103 /*
 104  * Struct.
 105  */
 106 struct field {
 107         char name[RTE_SWX_NAME_SIZE];
 108         uint32_t n_bits;
 109         uint32_t offset;
 110 };
 111
 112 struct struct_type {
 113         TAILQ_ENTRY(struct_type) node;
 114         char name[RTE_SWX_NAME_SIZE];
 115         struct field *fields;
 116         uint32_t n_fields;
 117         uint32_t n_bits;
 118 };
 119
 120 TAILQ_HEAD(struct_type_tailq, struct_type);
 121
 122 /*
 123  * Input port.
 124  */
 125 struct port_in_type {
 126         TAILQ_ENTRY(port_in_type) node;
 127         char name[RTE_SWX_NAME_SIZE];
 128         struct rte_swx_port_in_ops ops;
 129 };
 130
 131 TAILQ_HEAD(port_in_type_tailq, port_in_type);
 132
 133 struct port_in {
 134         TAILQ_ENTRY(port_in) node;
 135         struct port_in_type *type;
 136         void *obj;
 137         uint32_t id;
 138 };
 139
 140 TAILQ_HEAD(port_in_tailq, port_in);
 141
 142 struct port_in_runtime {
 143         rte_swx_port_in_pkt_rx_t pkt_rx;
 144         void *obj;
 145 };
 146
 147 /*
 148  * Output port.
 149  */
 150 struct port_out_type {
 151         TAILQ_ENTRY(port_out_type) node;
 152         char name[RTE_SWX_NAME_SIZE];
 153         struct rte_swx_port_out_ops ops;
 154 };
 155
 156 TAILQ_HEAD(port_out_type_tailq, port_out_type);
 157
 158 struct port_out {
 159         TAILQ_ENTRY(port_out) node;
 160         struct port_out_type *type;
 161         void *obj;
 162         uint32_t id;
 163 };
 164
 165 TAILQ_HEAD(port_out_tailq, port_out);
 166
 167 struct port_out_runtime {
 168         rte_swx_port_out_pkt_tx_t pkt_tx;
 169         rte_swx_port_out_flush_t flush;
 170         void *obj;
 171 };
 172
 173 /*
 174  * Extern object.
 175  */
 176 struct extern_type_member_func {
 177         TAILQ_ENTRY(extern_type_member_func) node;
 178         char name[RTE_SWX_NAME_SIZE];
 179         rte_swx_extern_type_member_func_t func;
 180         uint32_t id;
 181 };
 182
 183 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 184
 185 struct extern_type {
 186         TAILQ_ENTRY(extern_type) node;
 187         char name[RTE_SWX_NAME_SIZE];
 188         struct struct_type *mailbox_struct_type;
 189         rte_swx_extern_type_constructor_t constructor;
 190         rte_swx_extern_type_destructor_t destructor;
 191         struct extern_type_member_func_tailq funcs;
 192         uint32_t n_funcs;
 193 };
 194
 195 TAILQ_HEAD(extern_type_tailq, extern_type);
 196
 197 struct extern_obj {
 198         TAILQ_ENTRY(extern_obj) node;
 199         char name[RTE_SWX_NAME_SIZE];
 200         struct extern_type *type;
 201         void *obj;
 202         uint32_t struct_id;
 203         uint32_t id;
 204 };
 205
 206 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 207
 208 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 209 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 210 #endif
 211
 212 struct extern_obj_runtime {
 213         void *obj;
 214         uint8_t *mailbox;
 215         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 216 };
 217
 218 /*
 219  * Extern function.
 220  */
 221 struct extern_func {
 222         TAILQ_ENTRY(extern_func) node;
 223         char name[RTE_SWX_NAME_SIZE];
 224         struct struct_type *mailbox_struct_type;
 225         rte_swx_extern_func_t func;
 226         uint32_t struct_id;
 227         uint32_t id;
 228 };
 229
 230 TAILQ_HEAD(extern_func_tailq, extern_func);
 231
 232 struct extern_func_runtime {
 233         uint8_t *mailbox;
 234         rte_swx_extern_func_t func;
 235 };
 236
 237 /*
 238  * Header.
 239  */
 240 struct header {
 241         TAILQ_ENTRY(header) node;
 242         char name[RTE_SWX_NAME_SIZE];
 243         struct struct_type *st;
 244         uint32_t struct_id;
 245         uint32_t id;
 246 };
 247
 248 TAILQ_HEAD(header_tailq, header);
 249
 250 struct header_runtime {
 251         uint8_t *ptr0;
 252 };
 253
 254 struct header_out_runtime {
 255         uint8_t *ptr0;
 256         uint8_t *ptr;
 257         uint32_t n_bytes;
 258 };
 259
 260 /*
 261  * Instruction.
 262  */
 263
 264 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 265  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 266  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 267  * when transferred to packet meta-data and in NBO when transferred to packet
 268  * headers.
 269  */
 270
 271 /* Notation conventions:
 272  *    -Header field: H = h.header.field (dst/src)
 273  *    -Meta-data field: M = m.field (dst/src)
 274  *    -Extern object mailbox field: E = e.field (dst/src)
 275  *    -Extern function mailbox field: F = f.field (dst/src)
 276  *    -Table action data field: T = t.field (src only)
 277  *    -Immediate value: I = 32-bit unsigned value (src only)
 278  */
 279
 280 enum instruction_type {
 281         /* rx m.port_in */
 282         INSTR_RX,
 283
 284         /* tx port_out
 285          * port_out = MI
 286          */
 287         INSTR_TX,   /* port_out = M */
 288         INSTR_TX_I, /* port_out = I */
 289
 290         /* extract h.header */
 291         INSTR_HDR_EXTRACT,
 292         INSTR_HDR_EXTRACT2,
 293         INSTR_HDR_EXTRACT3,
 294         INSTR_HDR_EXTRACT4,
 295         INSTR_HDR_EXTRACT5,
 296         INSTR_HDR_EXTRACT6,
 297         INSTR_HDR_EXTRACT7,
 298         INSTR_HDR_EXTRACT8,
 299
 300         /* emit h.header */
 301         INSTR_HDR_EMIT,
 302         INSTR_HDR_EMIT_TX,
 303         INSTR_HDR_EMIT2_TX,
 304         INSTR_HDR_EMIT3_TX,
 305         INSTR_HDR_EMIT4_TX,
 306         INSTR_HDR_EMIT5_TX,
 307         INSTR_HDR_EMIT6_TX,
 308         INSTR_HDR_EMIT7_TX,
 309         INSTR_HDR_EMIT8_TX,
 310
 311         /* validate h.header */
 312         INSTR_HDR_VALIDATE,
 313
 314         /* invalidate h.header */
 315         INSTR_HDR_INVALIDATE,
 316
 317         /* mov dst src
 318          * dst = src
 319          * dst = HMEF, src = HMEFTI
 320          */
 321         INSTR_MOV,    /* dst = MEF, src = MEFT */
 322         INSTR_MOV_MH, /* dst = MEF, src = H */
 323         INSTR_MOV_HM, /* dst = H, src = MEFT */
 324         INSTR_MOV_HH, /* dst = H, src = H */
 325         INSTR_MOV_I,  /* dst = HMEF, src = I */
 326
 327         /* dma h.header t.field
 328          * memcpy(h.header, t.field, sizeof(h.header))
 329          */
 330         INSTR_DMA_HT,
 331         INSTR_DMA_HT2,
 332         INSTR_DMA_HT3,
 333         INSTR_DMA_HT4,
 334         INSTR_DMA_HT5,
 335         INSTR_DMA_HT6,
 336         INSTR_DMA_HT7,
 337         INSTR_DMA_HT8,
 338
 339         /* add dst src
 340          * dst += src
 341          * dst = HMEF, src = HMEFTI
 342          */
 343         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 344         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 345         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 346         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 347         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 348         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 349
 350         /* sub dst src
 351          * dst -= src
 352          * dst = HMEF, src = HMEFTI
 353          */
 354         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 355         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 356         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 357         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 358         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 359         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 360
 361         /* ckadd dst src
 362          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 363          * dst = H, src = {H, h.header}
 364          */
 365         INSTR_ALU_CKADD_FIELD,    /* src = H */
 366         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 367         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 368
 369         /* cksub dst src
 370          * dst = dst '- src
 371          * dst = H, src = H
 372          */
 373         INSTR_ALU_CKSUB_FIELD,
 374
 375         /* and dst src
 376          * dst &= src
 377          * dst = HMEF, src = HMEFTI
 378          */
 379         INSTR_ALU_AND,    /* dst = MEF, src = MEFT */
 380         INSTR_ALU_AND_MH, /* dst = MEF, src = H */
 381         INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
 382         INSTR_ALU_AND_HH, /* dst = H, src = H */
 383         INSTR_ALU_AND_I,  /* dst = HMEF, src = I */
 384
 385         /* or dst src
 386          * dst |= src
 387          * dst = HMEF, src = HMEFTI
 388          */
 389         INSTR_ALU_OR,    /* dst = MEF, src = MEFT */
 390         INSTR_ALU_OR_MH, /* dst = MEF, src = H */
 391         INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
 392         INSTR_ALU_OR_HH, /* dst = H, src = H */
 393         INSTR_ALU_OR_I,  /* dst = HMEF, src = I */
 394
 395         /* xor dst src
 396          * dst ^= src
 397          * dst = HMEF, src = HMEFTI
 398          */
 399         INSTR_ALU_XOR,    /* dst = MEF, src = MEFT */
 400         INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
 401         INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
 402         INSTR_ALU_XOR_HH, /* dst = H, src = H */
 403         INSTR_ALU_XOR_I,  /* dst = HMEF, src = I */
 404
 405         /* shl dst src
 406          * dst <<= src
 407          * dst = HMEF, src = HMEFTI
 408          */
 409         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 410         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 411         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 412         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 413         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 414         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 415
 416         /* shr dst src
 417          * dst >>= src
 418          * dst = HMEF, src = HMEFTI
 419          */
 420         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 421         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 422         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 423         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 424         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 425         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 426
 427         /* regprefetch REGARRAY index
 428          * prefetch REGARRAY[index]
 429          * index = HMEFTI
 430          */
 431         INSTR_REGPREFETCH_RH, /* index = H */
 432         INSTR_REGPREFETCH_RM, /* index = MEFT */
 433         INSTR_REGPREFETCH_RI, /* index = I */
 434
 435         /* regrd dst REGARRAY index
 436          * dst = REGARRAY[index]
 437          * dst = HMEF, index = HMEFTI
 438          */
 439         INSTR_REGRD_HRH, /* dst = H, index = H */
 440         INSTR_REGRD_HRM, /* dst = H, index = MEFT */
 441         INSTR_REGRD_HRI, /* dst = H, index = I */
 442         INSTR_REGRD_MRH, /* dst = MEF, index = H */
 443         INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
 444         INSTR_REGRD_MRI, /* dst = MEF, index = I */
 445
 446         /* regwr REGARRAY index src
 447          * REGARRAY[index] = src
 448          * index = HMEFTI, src = HMEFTI
 449          */
 450         INSTR_REGWR_RHH, /* index = H, src = H */
 451         INSTR_REGWR_RHM, /* index = H, src = MEFT */
 452         INSTR_REGWR_RHI, /* index = H, src = I */
 453         INSTR_REGWR_RMH, /* index = MEFT, src = H */
 454         INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
 455         INSTR_REGWR_RMI, /* index = MEFT, src = I */
 456         INSTR_REGWR_RIH, /* index = I, src = H */
 457         INSTR_REGWR_RIM, /* index = I, src = MEFT */
 458         INSTR_REGWR_RII, /* index = I, src = I */
 459
 460         /* regadd REGARRAY index src
 461          * REGARRAY[index] += src
 462          * index = HMEFTI, src = HMEFTI
 463          */
 464         INSTR_REGADD_RHH, /* index = H, src = H */
 465         INSTR_REGADD_RHM, /* index = H, src = MEFT */
 466         INSTR_REGADD_RHI, /* index = H, src = I */
 467         INSTR_REGADD_RMH, /* index = MEFT, src = H */
 468         INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
 469         INSTR_REGADD_RMI, /* index = MEFT, src = I */
 470         INSTR_REGADD_RIH, /* index = I, src = H */
 471         INSTR_REGADD_RIM, /* index = I, src = MEFT */
 472         INSTR_REGADD_RII, /* index = I, src = I */
 473
 474         /* metprefetch METARRAY index
 475          * prefetch METARRAY[index]
 476          * index = HMEFTI
 477          */
 478         INSTR_METPREFETCH_H, /* index = H */
 479         INSTR_METPREFETCH_M, /* index = MEFT */
 480         INSTR_METPREFETCH_I, /* index = I */
 481
 482         /* meter METARRAY index length color_in color_out
 483          * color_out = meter(METARRAY[index], length, color_in)
 484          * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
 485          */
 486         INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
 487         INSTR_METER_HHI, /* index = H, length = H, color_in = I */
 488         INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
 489         INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
 490         INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
 491         INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
 492         INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
 493         INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
 494         INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
 495         INSTR_METER_IHI, /* index = I, length = H, color_in = I */
 496         INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
 497         INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
 498
 499         /* table TABLE */
 500         INSTR_TABLE,
 501
 502         /* extern e.obj.func */
 503         INSTR_EXTERN_OBJ,
 504
 505         /* extern f.func */
 506         INSTR_EXTERN_FUNC,
 507
 508         /* jmp LABEL
 509          * Unconditional jump
 510          */
 511         INSTR_JMP,
 512
 513         /* jmpv LABEL h.header
 514          * Jump if header is valid
 515          */
 516         INSTR_JMP_VALID,
 517
 518         /* jmpnv LABEL h.header
 519          * Jump if header is invalid
 520          */
 521         INSTR_JMP_INVALID,
 522
 523         /* jmph LABEL
 524          * Jump if table lookup hit
 525          */
 526         INSTR_JMP_HIT,
 527
 528         /* jmpnh LABEL
 529          * Jump if table lookup miss
 530          */
 531         INSTR_JMP_MISS,
 532
 533         /* jmpa LABEL ACTION
 534          * Jump if action run
 535          */
 536         INSTR_JMP_ACTION_HIT,
 537
 538         /* jmpna LABEL ACTION
 539          * Jump if action not run
 540          */
 541         INSTR_JMP_ACTION_MISS,
 542
 543         /* jmpeq LABEL a b
 544          * Jump if a is equal to b
 545          * a = HMEFT, b = HMEFTI
 546          */
 547         INSTR_JMP_EQ,    /* a = MEFT, b = MEFT */
 548         INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
 549         INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
 550         INSTR_JMP_EQ_HH, /* a = H, b = H */
 551         INSTR_JMP_EQ_I,  /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 552
 553         /* jmpneq LABEL a b
 554          * Jump if a is not equal to b
 555          * a = HMEFT, b = HMEFTI
 556          */
 557         INSTR_JMP_NEQ,    /* a = MEFT, b = MEFT */
 558         INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
 559         INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
 560         INSTR_JMP_NEQ_HH, /* a = H, b = H */
 561         INSTR_JMP_NEQ_I,  /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 562
 563         /* jmplt LABEL a b
 564          * Jump if a is less than b
 565          * a = HMEFT, b = HMEFTI
 566          */
 567         INSTR_JMP_LT,    /* a = MEFT, b = MEFT */
 568         INSTR_JMP_LT_MH, /* a = MEFT, b = H */
 569         INSTR_JMP_LT_HM, /* a = H, b = MEFT */
 570         INSTR_JMP_LT_HH, /* a = H, b = H */
 571         INSTR_JMP_LT_MI, /* a = MEFT, b = I */
 572         INSTR_JMP_LT_HI, /* a = H, b = I */
 573
 574         /* jmpgt LABEL a b
 575          * Jump if a is greater than b
 576          * a = HMEFT, b = HMEFTI
 577          */
 578         INSTR_JMP_GT,    /* a = MEFT, b = MEFT */
 579         INSTR_JMP_GT_MH, /* a = MEFT, b = H */
 580         INSTR_JMP_GT_HM, /* a = H, b = MEFT */
 581         INSTR_JMP_GT_HH, /* a = H, b = H */
 582         INSTR_JMP_GT_MI, /* a = MEFT, b = I */
 583         INSTR_JMP_GT_HI, /* a = H, b = I */
 584
 585         /* return
 586          * Return from action
 587          */
 588         INSTR_RETURN,
 589 };
 590
 591 struct instr_operand {
 592         uint8_t struct_id;
 593         uint8_t n_bits;
 594         uint8_t offset;
 595         uint8_t pad;
 596 };
 597
 598 struct instr_io {
 599         struct {
 600                 union {
 601                         struct {
 602                                 uint8_t offset;
 603                                 uint8_t n_bits;
 604                                 uint8_t pad[2];
 605                         };
 606
 607                         uint32_t val;
 608                 };
 609         } io;
 610
 611         struct {
 612                 uint8_t header_id[8];
 613                 uint8_t struct_id[8];
 614                 uint8_t n_bytes[8];
 615         } hdr;
 616 };
 617
 618 struct instr_hdr_validity {
 619         uint8_t header_id;
 620 };
 621
 622 struct instr_table {
 623         uint8_t table_id;
 624 };
 625
 626 struct instr_extern_obj {
 627         uint8_t ext_obj_id;
 628         uint8_t func_id;
 629 };
 630
 631 struct instr_extern_func {
 632         uint8_t ext_func_id;
 633 };
 634
 635 struct instr_dst_src {
 636         struct instr_operand dst;
 637         union {
 638                 struct instr_operand src;
 639                 uint64_t src_val;
 640         };
 641 };
 642
 643 struct instr_regarray {
 644         uint8_t regarray_id;
 645         uint8_t pad[3];
 646
 647         union {
 648                 struct instr_operand idx;
 649                 uint32_t idx_val;
 650         };
 651
 652         union {
 653                 struct instr_operand dstsrc;
 654                 uint64_t dstsrc_val;
 655         };
 656 };
 657
 658 struct instr_meter {
 659         uint8_t metarray_id;
 660         uint8_t pad[3];
 661
 662         union {
 663                 struct instr_operand idx;
 664                 uint32_t idx_val;
 665         };
 666
 667         struct instr_operand length;
 668
 669         union {
 670                 struct instr_operand color_in;
 671                 uint32_t color_in_val;
 672         };
 673
 674         struct instr_operand color_out;
 675 };
 676
 677 struct instr_dma {
 678         struct {
 679                 uint8_t header_id[8];
 680                 uint8_t struct_id[8];
 681         } dst;
 682
 683         struct {
 684                 uint8_t offset[8];
 685         } src;
 686
 687         uint16_t n_bytes[8];
 688 };
 689
 690 struct instr_jmp {
 691         struct instruction *ip;
 692
 693         union {
 694                 struct instr_operand a;
 695                 uint8_t header_id;
 696                 uint8_t action_id;
 697         };
 698
 699         union {
 700                 struct instr_operand b;
 701                 uint64_t b_val;
 702         };
 703 };
 704
 705 struct instruction {
 706         enum instruction_type type;
 707         union {
 708                 struct instr_io io;
 709                 struct instr_hdr_validity valid;
 710                 struct instr_dst_src mov;
 711                 struct instr_regarray regarray;
 712                 struct instr_meter meter;
 713                 struct instr_dma dma;
 714                 struct instr_dst_src alu;
 715                 struct instr_table table;
 716                 struct instr_extern_obj ext_obj;
 717                 struct instr_extern_func ext_func;
 718                 struct instr_jmp jmp;
 719         };
 720 };
 721
 722 struct instruction_data {
 723         char label[RTE_SWX_NAME_SIZE];
 724         char jmp_label[RTE_SWX_NAME_SIZE];
 725         uint32_t n_users; /* user = jmp instruction to this instruction. */
 726         int invalid;
 727 };
 728
 729 /*
 730  * Action.
 731  */
 732 struct action {
 733         TAILQ_ENTRY(action) node;
 734         char name[RTE_SWX_NAME_SIZE];
 735         struct struct_type *st;
 736         struct instruction *instructions;
 737         uint32_t n_instructions;
 738         uint32_t id;
 739 };
 740
 741 TAILQ_HEAD(action_tailq, action);
 742
 743 /*
 744  * Table.
 745  */
 746 struct table_type {
 747         TAILQ_ENTRY(table_type) node;
 748         char name[RTE_SWX_NAME_SIZE];
 749         enum rte_swx_table_match_type match_type;
 750         struct rte_swx_table_ops ops;
 751 };
 752
 753 TAILQ_HEAD(table_type_tailq, table_type);
 754
 755 struct match_field {
 756         enum rte_swx_table_match_type match_type;
 757         struct field *field;
 758 };
 759
 760 struct table {
 761         TAILQ_ENTRY(table) node;
 762         char name[RTE_SWX_NAME_SIZE];
 763         char args[RTE_SWX_NAME_SIZE];
 764         struct table_type *type; /* NULL when n_fields == 0. */
 765
 766         /* Match. */
 767         struct match_field *fields;
 768         uint32_t n_fields;
 769         struct header *header; /* Only valid when n_fields > 0. */
 770
 771         /* Action. */
 772         struct action **actions;
 773         struct action *default_action;
 774         uint8_t *default_action_data;
 775         uint32_t n_actions;
 776         int default_action_is_const;
 777         uint32_t action_data_size_max;
 778
 779         uint32_t size;
 780         uint32_t id;
 781 };
 782
 783 TAILQ_HEAD(table_tailq, table);
 784
 785 struct table_runtime {
 786         rte_swx_table_lookup_t func;
 787         void *mailbox;
 788         uint8_t **key;
 789 };
 790
 791 struct table_statistics {
 792         uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
 793         uint64_t *n_pkts_action;
 794 };
 795
 796 /*
 797  * Register array.
 798  */
 799 struct regarray {
 800         TAILQ_ENTRY(regarray) node;
 801         char name[RTE_SWX_NAME_SIZE];
 802         uint64_t init_val;
 803         uint32_t size;
 804         uint32_t id;
 805 };
 806
 807 TAILQ_HEAD(regarray_tailq, regarray);
 808
 809 struct regarray_runtime {
 810         uint64_t *regarray;
 811         uint32_t size_mask;
 812 };
 813
 814 /*
 815  * Meter array.
 816  */
 817 struct meter_profile {
 818         TAILQ_ENTRY(meter_profile) node;
 819         char name[RTE_SWX_NAME_SIZE];
 820         struct rte_meter_trtcm_params params;
 821         struct rte_meter_trtcm_profile profile;
 822         uint32_t n_users;
 823 };
 824
 825 TAILQ_HEAD(meter_profile_tailq, meter_profile);
 826
 827 struct metarray {
 828         TAILQ_ENTRY(metarray) node;
 829         char name[RTE_SWX_NAME_SIZE];
 830         uint32_t size;
 831         uint32_t id;
 832 };
 833
 834 TAILQ_HEAD(metarray_tailq, metarray);
 835
 836 struct meter {
 837         struct rte_meter_trtcm m;
 838         struct meter_profile *profile;
 839         enum rte_color color_mask;
 840         uint8_t pad[20];
 841
 842         uint64_t n_pkts[RTE_COLORS];
 843         uint64_t n_bytes[RTE_COLORS];
 844 };
 845
 846 struct metarray_runtime {
 847         struct meter *metarray;
 848         uint32_t size_mask;
 849 };
 850
 851 /*
 852  * Pipeline.
 853  */
 854 struct thread {
 855         /* Packet. */
 856         struct rte_swx_pkt pkt;
 857         uint8_t *ptr;
 858
 859         /* Structures. */
 860         uint8_t **structs;
 861
 862         /* Packet headers. */
 863         struct header_runtime *headers; /* Extracted or generated headers. */
 864         struct header_out_runtime *headers_out; /* Emitted headers. */
 865         uint8_t *header_storage;
 866         uint8_t *header_out_storage;
 867         uint64_t valid_headers;
 868         uint32_t n_headers_out;
 869
 870         /* Packet meta-data. */
 871         uint8_t *metadata;
 872
 873         /* Tables. */
 874         struct table_runtime *tables;
 875         struct rte_swx_table_state *table_state;
 876         uint64_t action_id;
 877         int hit; /* 0 = Miss, 1 = Hit. */
 878
 879         /* Extern objects and functions. */
 880         struct extern_obj_runtime *extern_objs;
 881         struct extern_func_runtime *extern_funcs;
 882
 883         /* Instructions. */
 884         struct instruction *ip;
 885         struct instruction *ret;
 886 };
 887
 888 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 889 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 890 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 891
 892 #define HEADER_VALID(thread, header_id) \
 893         MASK64_BIT_GET((thread)->valid_headers, header_id)
 894
 895 #define ALU(thread, ip, operator)  \
 896 {                                                                              \
 897         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 898         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 899         uint64_t dst64 = *dst64_ptr;                                           \
 900         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 901         uint64_t dst = dst64 & dst64_mask;                                     \
 902                                                                                \
 903         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 904         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 905         uint64_t src64 = *src64_ptr;                                           \
 906         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 907         uint64_t src = src64 & src64_mask;                                     \
 908                                                                                \
 909         uint64_t result = dst operator src;                                    \
 910                                                                                \
 911         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 912 }
 913
 914 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 915
 916 #define ALU_MH(thread, ip, operator)  \
 917 {                                                                              \
 918         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 919         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 920         uint64_t dst64 = *dst64_ptr;                                           \
 921         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 922         uint64_t dst = dst64 & dst64_mask;                                     \
 923                                                                                \
 924         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 925         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 926         uint64_t src64 = *src64_ptr;                                           \
 927         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 928                                                                                \
 929         uint64_t result = dst operator src;                                    \
 930                                                                                \
 931         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 932 }
 933
 934 #define ALU_HM(thread, ip, operator)  \
 935 {                                                                              \
 936         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 937         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 938         uint64_t dst64 = *dst64_ptr;                                           \
 939         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 940         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 941                                                                                \
 942         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 943         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 944         uint64_t src64 = *src64_ptr;                                           \
 945         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 946         uint64_t src = src64 & src64_mask;                                     \
 947                                                                                \
 948         uint64_t result = dst operator src;                                    \
 949         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 950                                                                                \
 951         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 952 }
 953
 954 #define ALU_HM_FAST(thread, ip, operator)  \
 955 {                                                                                 \
 956         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];         \
 957         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];      \
 958         uint64_t dst64 = *dst64_ptr;                                              \
 959         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);          \
 960         uint64_t dst = dst64 & dst64_mask;                                        \
 961                                                                                   \
 962         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];         \
 963         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];      \
 964         uint64_t src64 = *src64_ptr;                                              \
 965         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);          \
 966         uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
 967                                                                                   \
 968         uint64_t result = dst operator src;                                       \
 969                                                                                   \
 970         *dst64_ptr = (dst64 & ~dst64_mask) | result;                              \
 971 }
 972
 973 #define ALU_HH(thread, ip, operator)  \
 974 {                                                                              \
 975         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 976         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 977         uint64_t dst64 = *dst64_ptr;                                           \
 978         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 979         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 980                                                                                \
 981         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 982         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 983         uint64_t src64 = *src64_ptr;                                           \
 984         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 985                                                                                \
 986         uint64_t result = dst operator src;                                    \
 987         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 988                                                                                \
 989         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 990 }
 991
 992 #define ALU_HH_FAST(thread, ip, operator)  \
 993 {                                                                                             \
 994         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];                     \
 995         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];                  \
 996         uint64_t dst64 = *dst64_ptr;                                                          \
 997         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);                      \
 998         uint64_t dst = dst64 & dst64_mask;                                                    \
 999                                                                                               \
1000         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];                     \
1001         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];                  \
1002         uint64_t src64 = *src64_ptr;                                                          \
1003         uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1004                                                                                               \
1005         uint64_t result = dst operator src;                                                   \
1006                                                                                               \
1007         *dst64_ptr = (dst64 & ~dst64_mask) | result;                                          \
1008 }
1009
1010 #else
1011
1012 #define ALU_MH ALU
1013 #define ALU_HM ALU
1014 #define ALU_HM_FAST ALU
1015 #define ALU_HH ALU
1016 #define ALU_HH_FAST ALU
1017
1018 #endif
1019
1020 #define ALU_I(thread, ip, operator)  \
1021 {                                                                              \
1022         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1023         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1024         uint64_t dst64 = *dst64_ptr;                                           \
1025         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1026         uint64_t dst = dst64 & dst64_mask;                                     \
1027                                                                                \
1028         uint64_t src = (ip)->alu.src_val;                                      \
1029                                                                                \
1030         uint64_t result = dst operator src;                                    \
1031                                                                                \
1032         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
1033 }
1034
1035 #define ALU_MI ALU_I
1036
1037 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1038
1039 #define ALU_HI(thread, ip, operator)  \
1040 {                                                                              \
1041         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1042         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1043         uint64_t dst64 = *dst64_ptr;                                           \
1044         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1045         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
1046                                                                                \
1047         uint64_t src = (ip)->alu.src_val;                                      \
1048                                                                                \
1049         uint64_t result = dst operator src;                                    \
1050         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1051                                                                                \
1052         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1053 }
1054
1055 #else
1056
1057 #define ALU_HI ALU_I
1058
1059 #endif
1060
1061 #define MOV(thread, ip)  \
1062 {                                                                              \
1063         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1064         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1065         uint64_t dst64 = *dst64_ptr;                                           \
1066         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1067                                                                                \
1068         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1069         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1070         uint64_t src64 = *src64_ptr;                                           \
1071         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1072         uint64_t src = src64 & src64_mask;                                     \
1073                                                                                \
1074         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1075 }
1076
1077 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1078
1079 #define MOV_MH(thread, ip)  \
1080 {                                                                              \
1081         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1082         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1083         uint64_t dst64 = *dst64_ptr;                                           \
1084         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1085                                                                                \
1086         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1087         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1088         uint64_t src64 = *src64_ptr;                                           \
1089         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
1090                                                                                \
1091         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1092 }
1093
1094 #define MOV_HM(thread, ip)  \
1095 {                                                                              \
1096         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1097         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1098         uint64_t dst64 = *dst64_ptr;                                           \
1099         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1100                                                                                \
1101         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1102         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1103         uint64_t src64 = *src64_ptr;                                           \
1104         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1105         uint64_t src = src64 & src64_mask;                                     \
1106                                                                                \
1107         src = hton64(src) >> (64 - (ip)->mov.dst.n_bits);                      \
1108         *dst64_ptr = (dst64 & ~dst64_mask) | src;                              \
1109 }
1110
1111 #define MOV_HH(thread, ip)  \
1112 {                                                                              \
1113         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1114         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1115         uint64_t dst64 = *dst64_ptr;                                           \
1116         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1117                                                                                \
1118         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1119         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1120         uint64_t src64 = *src64_ptr;                                           \
1121                                                                                \
1122         uint64_t src = src64 << (64 - (ip)->mov.src.n_bits);                   \
1123         src = src >> (64 - (ip)->mov.dst.n_bits);                              \
1124         *dst64_ptr = (dst64 & ~dst64_mask) | src;                              \
1125 }
1126
1127 #else
1128
1129 #define MOV_MH MOV
1130 #define MOV_HM MOV
1131 #define MOV_HH MOV
1132
1133 #endif
1134
1135 #define MOV_I(thread, ip)  \
1136 {                                                                              \
1137         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1138         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1139         uint64_t dst64 = *dst64_ptr;                                           \
1140         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1141                                                                                \
1142         uint64_t src = (ip)->mov.src_val;                                      \
1143                                                                                \
1144         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1145 }
1146
1147 #define JMP_CMP(thread, ip, operator)  \
1148 {                                                                              \
1149         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1150         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1151         uint64_t a64 = *a64_ptr;                                               \
1152         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1153         uint64_t a = a64 & a64_mask;                                           \
1154                                                                                \
1155         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1156         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1157         uint64_t b64 = *b64_ptr;                                               \
1158         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1159         uint64_t b = b64 & b64_mask;                                           \
1160                                                                                \
1161         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1162 }
1163
1164 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1165
1166 #define JMP_CMP_MH(thread, ip, operator)  \
1167 {                                                                              \
1168         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1169         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1170         uint64_t a64 = *a64_ptr;                                               \
1171         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1172         uint64_t a = a64 & a64_mask;                                           \
1173                                                                                \
1174         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1175         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1176         uint64_t b64 = *b64_ptr;                                               \
1177         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1178                                                                                \
1179         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1180 }
1181
1182 #define JMP_CMP_HM(thread, ip, operator)  \
1183 {                                                                              \
1184         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1185         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1186         uint64_t a64 = *a64_ptr;                                               \
1187         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1188                                                                                \
1189         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1190         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1191         uint64_t b64 = *b64_ptr;                                               \
1192         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1193         uint64_t b = b64 & b64_mask;                                           \
1194                                                                                \
1195         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1196 }
1197
1198 #define JMP_CMP_HH(thread, ip, operator)  \
1199 {                                                                              \
1200         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1201         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1202         uint64_t a64 = *a64_ptr;                                               \
1203         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1204                                                                                \
1205         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1206         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1207         uint64_t b64 = *b64_ptr;                                               \
1208         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1209                                                                                \
1210         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1211 }
1212
1213 #define JMP_CMP_HH_FAST(thread, ip, operator)  \
1214 {                                                                              \
1215         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1216         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1217         uint64_t a64 = *a64_ptr;                                               \
1218         uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits);                         \
1219                                                                                \
1220         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1221         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1222         uint64_t b64 = *b64_ptr;                                               \
1223         uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits);                         \
1224                                                                                \
1225         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1226 }
1227
1228 #else
1229
1230 #define JMP_CMP_MH JMP_CMP
1231 #define JMP_CMP_HM JMP_CMP
1232 #define JMP_CMP_HH JMP_CMP
1233 #define JMP_CMP_HH_FAST JMP_CMP
1234
1235 #endif
1236
1237 #define JMP_CMP_I(thread, ip, operator)  \
1238 {                                                                              \
1239         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1240         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1241         uint64_t a64 = *a64_ptr;                                               \
1242         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1243         uint64_t a = a64 & a64_mask;                                           \
1244                                                                                \
1245         uint64_t b = (ip)->jmp.b_val;                                          \
1246                                                                                \
1247         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1248 }
1249
1250 #define JMP_CMP_MI JMP_CMP_I
1251
1252 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1253
1254 #define JMP_CMP_HI(thread, ip, operator)  \
1255 {                                                                              \
1256         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1257         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1258         uint64_t a64 = *a64_ptr;                                               \
1259         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1260                                                                                \
1261         uint64_t b = (ip)->jmp.b_val;                                          \
1262                                                                                \
1263         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1264 }
1265
1266 #else
1267
1268 #define JMP_CMP_HI JMP_CMP_I
1269
1270 #endif
1271
1272 #define METADATA_READ(thread, offset, n_bits)                                  \
1273 ({                                                                             \
1274         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1275         uint64_t m64 = *m64_ptr;                                               \
1276         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1277         (m64 & m64_mask);                                                      \
1278 })
1279
1280 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
1281 {                                                                              \
1282         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1283         uint64_t m64 = *m64_ptr;                                               \
1284         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1285                                                                                \
1286         uint64_t m_new = value;                                                \
1287                                                                                \
1288         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
1289 }
1290
1291 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1292 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1293 #endif
1294
1295 struct rte_swx_pipeline {
1296         struct struct_type_tailq struct_types;
1297         struct port_in_type_tailq port_in_types;
1298         struct port_in_tailq ports_in;
1299         struct port_out_type_tailq port_out_types;
1300         struct port_out_tailq ports_out;
1301         struct extern_type_tailq extern_types;
1302         struct extern_obj_tailq extern_objs;
1303         struct extern_func_tailq extern_funcs;
1304         struct header_tailq headers;
1305         struct struct_type *metadata_st;
1306         uint32_t metadata_struct_id;
1307         struct action_tailq actions;
1308         struct table_type_tailq table_types;
1309         struct table_tailq tables;
1310         struct regarray_tailq regarrays;
1311         struct meter_profile_tailq meter_profiles;
1312         struct metarray_tailq metarrays;
1313
1314         struct port_in_runtime *in;
1315         struct port_out_runtime *out;
1316         struct instruction **action_instructions;
1317         struct rte_swx_table_state *table_state;
1318         struct table_statistics *table_stats;
1319         struct regarray_runtime *regarray_runtime;
1320         struct metarray_runtime *metarray_runtime;
1321         struct instruction *instructions;
1322         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1323
1324         uint32_t n_structs;
1325         uint32_t n_ports_in;
1326         uint32_t n_ports_out;
1327         uint32_t n_extern_objs;
1328         uint32_t n_extern_funcs;
1329         uint32_t n_actions;
1330         uint32_t n_tables;
1331         uint32_t n_regarrays;
1332         uint32_t n_metarrays;
1333         uint32_t n_headers;
1334         uint32_t thread_id;
1335         uint32_t port_id;
1336         uint32_t n_instructions;
1337         int build_done;
1338         int numa_node;
1339 };
1340
1341 /*
1342  * Struct.
1343  */
1344 static struct struct_type *
1345 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1346 {
1347         struct struct_type *elem;
1348
1349         TAILQ_FOREACH(elem, &p->struct_types, node)
1350                 if (strcmp(elem->name, name) == 0)
1351                         return elem;
1352
1353         return NULL;
1354 }
1355
1356 static struct field *
1357 struct_type_field_find(struct struct_type *st, const char *name)
1358 {
1359         uint32_t i;
1360
1361         for (i = 0; i < st->n_fields; i++) {
1362                 struct field *f = &st->fields[i];
1363
1364                 if (strcmp(f->name, name) == 0)
1365                         return f;
1366         }
1367
1368         return NULL;
1369 }
1370
1371 int
1372 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1373                                       const char *name,
1374                                       struct rte_swx_field_params *fields,
1375                                       uint32_t n_fields)
1376 {
1377         struct struct_type *st;
1378         uint32_t i;
1379
1380         CHECK(p, EINVAL);
1381         CHECK_NAME(name, EINVAL);
1382         CHECK(fields, EINVAL);
1383         CHECK(n_fields, EINVAL);
1384
1385         for (i = 0; i < n_fields; i++) {
1386                 struct rte_swx_field_params *f = &fields[i];
1387                 uint32_t j;
1388
1389                 CHECK_NAME(f->name, EINVAL);
1390                 CHECK(f->n_bits, EINVAL);
1391                 CHECK(f->n_bits <= 64, EINVAL);
1392                 CHECK((f->n_bits & 7) == 0, EINVAL);
1393
1394                 for (j = 0; j < i; j++) {
1395                         struct rte_swx_field_params *f_prev = &fields[j];
1396
1397                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
1398                 }
1399         }
1400
1401         CHECK(!struct_type_find(p, name), EEXIST);
1402
1403         /* Node allocation. */
1404         st = calloc(1, sizeof(struct struct_type));
1405         CHECK(st, ENOMEM);
1406
1407         st->fields = calloc(n_fields, sizeof(struct field));
1408         if (!st->fields) {
1409                 free(st);
1410                 CHECK(0, ENOMEM);
1411         }
1412
1413         /* Node initialization. */
1414         strcpy(st->name, name);
1415         for (i = 0; i < n_fields; i++) {
1416                 struct field *dst = &st->fields[i];
1417                 struct rte_swx_field_params *src = &fields[i];
1418
1419                 strcpy(dst->name, src->name);
1420                 dst->n_bits = src->n_bits;
1421                 dst->offset = st->n_bits;
1422
1423                 st->n_bits += src->n_bits;
1424         }
1425         st->n_fields = n_fields;
1426
1427         /* Node add to tailq. */
1428         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1429
1430         return 0;
1431 }
1432
1433 static int
1434 struct_build(struct rte_swx_pipeline *p)
1435 {
1436         uint32_t i;
1437
1438         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1439                 struct thread *t = &p->threads[i];
1440
1441                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1442                 CHECK(t->structs, ENOMEM);
1443         }
1444
1445         return 0;
1446 }
1447
1448 static void
1449 struct_build_free(struct rte_swx_pipeline *p)
1450 {
1451         uint32_t i;
1452
1453         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1454                 struct thread *t = &p->threads[i];
1455
1456                 free(t->structs);
1457                 t->structs = NULL;
1458         }
1459 }
1460
1461 static void
1462 struct_free(struct rte_swx_pipeline *p)
1463 {
1464         struct_build_free(p);
1465
1466         /* Struct types. */
1467         for ( ; ; ) {
1468                 struct struct_type *elem;
1469
1470                 elem = TAILQ_FIRST(&p->struct_types);
1471                 if (!elem)
1472                         break;
1473
1474                 TAILQ_REMOVE(&p->struct_types, elem, node);
1475                 free(elem->fields);
1476                 free(elem);
1477         }
1478 }
1479
1480 /*
1481  * Input port.
1482  */
1483 static struct port_in_type *
1484 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1485 {
1486         struct port_in_type *elem;
1487
1488         if (!name)
1489                 return NULL;
1490
1491         TAILQ_FOREACH(elem, &p->port_in_types, node)
1492                 if (strcmp(elem->name, name) == 0)
1493                         return elem;
1494
1495         return NULL;
1496 }
1497
1498 int
1499 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1500                                        const char *name,
1501                                        struct rte_swx_port_in_ops *ops)
1502 {
1503         struct port_in_type *elem;
1504
1505         CHECK(p, EINVAL);
1506         CHECK_NAME(name, EINVAL);
1507         CHECK(ops, EINVAL);
1508         CHECK(ops->create, EINVAL);
1509         CHECK(ops->free, EINVAL);
1510         CHECK(ops->pkt_rx, EINVAL);
1511         CHECK(ops->stats_read, EINVAL);
1512
1513         CHECK(!port_in_type_find(p, name), EEXIST);
1514
1515         /* Node allocation. */
1516         elem = calloc(1, sizeof(struct port_in_type));
1517         CHECK(elem, ENOMEM);
1518
1519         /* Node initialization. */
1520         strcpy(elem->name, name);
1521         memcpy(&elem->ops, ops, sizeof(*ops));
1522
1523         /* Node add to tailq. */
1524         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1525
1526         return 0;
1527 }
1528
1529 static struct port_in *
1530 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1531 {
1532         struct port_in *port;
1533
1534         TAILQ_FOREACH(port, &p->ports_in, node)
1535                 if (port->id == port_id)
1536                         return port;
1537
1538         return NULL;
1539 }
1540
1541 int
1542 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1543                                 uint32_t port_id,
1544                                 const char *port_type_name,
1545                                 void *args)
1546 {
1547         struct port_in_type *type = NULL;
1548         struct port_in *port = NULL;
1549         void *obj = NULL;
1550
1551         CHECK(p, EINVAL);
1552
1553         CHECK(!port_in_find(p, port_id), EINVAL);
1554
1555         CHECK_NAME(port_type_name, EINVAL);
1556         type = port_in_type_find(p, port_type_name);
1557         CHECK(type, EINVAL);
1558
1559         obj = type->ops.create(args);
1560         CHECK(obj, ENODEV);
1561
1562         /* Node allocation. */
1563         port = calloc(1, sizeof(struct port_in));
1564         CHECK(port, ENOMEM);
1565
1566         /* Node initialization. */
1567         port->type = type;
1568         port->obj = obj;
1569         port->id = port_id;
1570
1571         /* Node add to tailq. */
1572         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1573         if (p->n_ports_in < port_id + 1)
1574                 p->n_ports_in = port_id + 1;
1575
1576         return 0;
1577 }
1578
1579 static int
1580 port_in_build(struct rte_swx_pipeline *p)
1581 {
1582         struct port_in *port;
1583         uint32_t i;
1584
1585         CHECK(p->n_ports_in, EINVAL);
1586         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1587
1588         for (i = 0; i < p->n_ports_in; i++)
1589                 CHECK(port_in_find(p, i), EINVAL);
1590
1591         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1592         CHECK(p->in, ENOMEM);
1593
1594         TAILQ_FOREACH(port, &p->ports_in, node) {
1595                 struct port_in_runtime *in = &p->in[port->id];
1596
1597                 in->pkt_rx = port->type->ops.pkt_rx;
1598                 in->obj = port->obj;
1599         }
1600
1601         return 0;
1602 }
1603
1604 static void
1605 port_in_build_free(struct rte_swx_pipeline *p)
1606 {
1607         free(p->in);
1608         p->in = NULL;
1609 }
1610
1611 static void
1612 port_in_free(struct rte_swx_pipeline *p)
1613 {
1614         port_in_build_free(p);
1615
1616         /* Input ports. */
1617         for ( ; ; ) {
1618                 struct port_in *port;
1619
1620                 port = TAILQ_FIRST(&p->ports_in);
1621                 if (!port)
1622                         break;
1623
1624                 TAILQ_REMOVE(&p->ports_in, port, node);
1625                 port->type->ops.free(port->obj);
1626                 free(port);
1627         }
1628
1629         /* Input port types. */
1630         for ( ; ; ) {
1631                 struct port_in_type *elem;
1632
1633                 elem = TAILQ_FIRST(&p->port_in_types);
1634                 if (!elem)
1635                         break;
1636
1637                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1638                 free(elem);
1639         }
1640 }
1641
1642 /*
1643  * Output port.
1644  */
1645 static struct port_out_type *
1646 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1647 {
1648         struct port_out_type *elem;
1649
1650         if (!name)
1651                 return NULL;
1652
1653         TAILQ_FOREACH(elem, &p->port_out_types, node)
1654                 if (!strcmp(elem->name, name))
1655                         return elem;
1656
1657         return NULL;
1658 }
1659
1660 int
1661 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1662                                         const char *name,
1663                                         struct rte_swx_port_out_ops *ops)
1664 {
1665         struct port_out_type *elem;
1666
1667         CHECK(p, EINVAL);
1668         CHECK_NAME(name, EINVAL);
1669         CHECK(ops, EINVAL);
1670         CHECK(ops->create, EINVAL);
1671         CHECK(ops->free, EINVAL);
1672         CHECK(ops->pkt_tx, EINVAL);
1673         CHECK(ops->stats_read, EINVAL);
1674
1675         CHECK(!port_out_type_find(p, name), EEXIST);
1676
1677         /* Node allocation. */
1678         elem = calloc(1, sizeof(struct port_out_type));
1679         CHECK(elem, ENOMEM);
1680
1681         /* Node initialization. */
1682         strcpy(elem->name, name);
1683         memcpy(&elem->ops, ops, sizeof(*ops));
1684
1685         /* Node add to tailq. */
1686         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1687
1688         return 0;
1689 }
1690
1691 static struct port_out *
1692 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1693 {
1694         struct port_out *port;
1695
1696         TAILQ_FOREACH(port, &p->ports_out, node)
1697                 if (port->id == port_id)
1698                         return port;
1699
1700         return NULL;
1701 }
1702
1703 int
1704 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1705                                  uint32_t port_id,
1706                                  const char *port_type_name,
1707                                  void *args)
1708 {
1709         struct port_out_type *type = NULL;
1710         struct port_out *port = NULL;
1711         void *obj = NULL;
1712
1713         CHECK(p, EINVAL);
1714
1715         CHECK(!port_out_find(p, port_id), EINVAL);
1716
1717         CHECK_NAME(port_type_name, EINVAL);
1718         type = port_out_type_find(p, port_type_name);
1719         CHECK(type, EINVAL);
1720
1721         obj = type->ops.create(args);
1722         CHECK(obj, ENODEV);
1723
1724         /* Node allocation. */
1725         port = calloc(1, sizeof(struct port_out));
1726         CHECK(port, ENOMEM);
1727
1728         /* Node initialization. */
1729         port->type = type;
1730         port->obj = obj;
1731         port->id = port_id;
1732
1733         /* Node add to tailq. */
1734         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1735         if (p->n_ports_out < port_id + 1)
1736                 p->n_ports_out = port_id + 1;
1737
1738         return 0;
1739 }
1740
1741 static int
1742 port_out_build(struct rte_swx_pipeline *p)
1743 {
1744         struct port_out *port;
1745         uint32_t i;
1746
1747         CHECK(p->n_ports_out, EINVAL);
1748
1749         for (i = 0; i < p->n_ports_out; i++)
1750                 CHECK(port_out_find(p, i), EINVAL);
1751
1752         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1753         CHECK(p->out, ENOMEM);
1754
1755         TAILQ_FOREACH(port, &p->ports_out, node) {
1756                 struct port_out_runtime *out = &p->out[port->id];
1757
1758                 out->pkt_tx = port->type->ops.pkt_tx;
1759                 out->flush = port->type->ops.flush;
1760                 out->obj = port->obj;
1761         }
1762
1763         return 0;
1764 }
1765
1766 static void
1767 port_out_build_free(struct rte_swx_pipeline *p)
1768 {
1769         free(p->out);
1770         p->out = NULL;
1771 }
1772
1773 static void
1774 port_out_free(struct rte_swx_pipeline *p)
1775 {
1776         port_out_build_free(p);
1777
1778         /* Output ports. */
1779         for ( ; ; ) {
1780                 struct port_out *port;
1781
1782                 port = TAILQ_FIRST(&p->ports_out);
1783                 if (!port)
1784                         break;
1785
1786                 TAILQ_REMOVE(&p->ports_out, port, node);
1787                 port->type->ops.free(port->obj);
1788                 free(port);
1789         }
1790
1791         /* Output port types. */
1792         for ( ; ; ) {
1793                 struct port_out_type *elem;
1794
1795                 elem = TAILQ_FIRST(&p->port_out_types);
1796                 if (!elem)
1797                         break;
1798
1799                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1800                 free(elem);
1801         }
1802 }
1803
1804 /*
1805  * Extern object.
1806  */
1807 static struct extern_type *
1808 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1809 {
1810         struct extern_type *elem;
1811
1812         TAILQ_FOREACH(elem, &p->extern_types, node)
1813                 if (strcmp(elem->name, name) == 0)
1814                         return elem;
1815
1816         return NULL;
1817 }
1818
1819 static struct extern_type_member_func *
1820 extern_type_member_func_find(struct extern_type *type, const char *name)
1821 {
1822         struct extern_type_member_func *elem;
1823
1824         TAILQ_FOREACH(elem, &type->funcs, node)
1825                 if (strcmp(elem->name, name) == 0)
1826                         return elem;
1827
1828         return NULL;
1829 }
1830
1831 static struct extern_obj *
1832 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1833 {
1834         struct extern_obj *elem;
1835
1836         TAILQ_FOREACH(elem, &p->extern_objs, node)
1837                 if (strcmp(elem->name, name) == 0)
1838                         return elem;
1839
1840         return NULL;
1841 }
1842
1843 static struct extern_type_member_func *
1844 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1845                              const char *name,
1846                              struct extern_obj **obj)
1847 {
1848         struct extern_obj *object;
1849         struct extern_type_member_func *func;
1850         char *object_name, *func_name;
1851
1852         if (name[0] != 'e' || name[1] != '.')
1853                 return NULL;
1854
1855         object_name = strdup(&name[2]);
1856         if (!object_name)
1857                 return NULL;
1858
1859         func_name = strchr(object_name, '.');
1860         if (!func_name) {
1861                 free(object_name);
1862                 return NULL;
1863         }
1864
1865         *func_name = 0;
1866         func_name++;
1867
1868         object = extern_obj_find(p, object_name);
1869         if (!object) {
1870                 free(object_name);
1871                 return NULL;
1872         }
1873
1874         func = extern_type_member_func_find(object->type, func_name);
1875         if (!func) {
1876                 free(object_name);
1877                 return NULL;
1878         }
1879
1880         if (obj)
1881                 *obj = object;
1882
1883         free(object_name);
1884         return func;
1885 }
1886
1887 static struct field *
1888 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1889                                const char *name,
1890                                struct extern_obj **object)
1891 {
1892         struct extern_obj *obj;
1893         struct field *f;
1894         char *obj_name, *field_name;
1895
1896         if ((name[0] != 'e') || (name[1] != '.'))
1897                 return NULL;
1898
1899         obj_name = strdup(&name[2]);
1900         if (!obj_name)
1901                 return NULL;
1902
1903         field_name = strchr(obj_name, '.');
1904         if (!field_name) {
1905                 free(obj_name);
1906                 return NULL;
1907         }
1908
1909         *field_name = 0;
1910         field_name++;
1911
1912         obj = extern_obj_find(p, obj_name);
1913         if (!obj) {
1914                 free(obj_name);
1915                 return NULL;
1916         }
1917
1918         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1919         if (!f) {
1920                 free(obj_name);
1921                 return NULL;
1922         }
1923
1924         if (object)
1925                 *object = obj;
1926
1927         free(obj_name);
1928         return f;
1929 }
1930
1931 int
1932 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1933         const char *name,
1934         const char *mailbox_struct_type_name,
1935         rte_swx_extern_type_constructor_t constructor,
1936         rte_swx_extern_type_destructor_t destructor)
1937 {
1938         struct extern_type *elem;
1939         struct struct_type *mailbox_struct_type;
1940
1941         CHECK(p, EINVAL);
1942
1943         CHECK_NAME(name, EINVAL);
1944         CHECK(!extern_type_find(p, name), EEXIST);
1945
1946         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1947         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1948         CHECK(mailbox_struct_type, EINVAL);
1949
1950         CHECK(constructor, EINVAL);
1951         CHECK(destructor, EINVAL);
1952
1953         /* Node allocation. */
1954         elem = calloc(1, sizeof(struct extern_type));
1955         CHECK(elem, ENOMEM);
1956
1957         /* Node initialization. */
1958         strcpy(elem->name, name);
1959         elem->mailbox_struct_type = mailbox_struct_type;
1960         elem->constructor = constructor;
1961         elem->destructor = destructor;
1962         TAILQ_INIT(&elem->funcs);
1963
1964         /* Node add to tailq. */
1965         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1966
1967         return 0;
1968 }
1969
1970 int
1971 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1972         const char *extern_type_name,
1973         const char *name,
1974         rte_swx_extern_type_member_func_t member_func)
1975 {
1976         struct extern_type *type;
1977         struct extern_type_member_func *type_member;
1978
1979         CHECK(p, EINVAL);
1980
1981         CHECK_NAME(extern_type_name, EINVAL);
1982         type = extern_type_find(p, extern_type_name);
1983         CHECK(type, EINVAL);
1984         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1985
1986         CHECK_NAME(name, EINVAL);
1987         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1988
1989         CHECK(member_func, EINVAL);
1990
1991         /* Node allocation. */
1992         type_member = calloc(1, sizeof(struct extern_type_member_func));
1993         CHECK(type_member, ENOMEM);
1994
1995         /* Node initialization. */
1996         strcpy(type_member->name, name);
1997         type_member->func = member_func;
1998         type_member->id = type->n_funcs;
1999
2000         /* Node add to tailq. */
2001         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
2002         type->n_funcs++;
2003
2004         return 0;
2005 }
2006
2007 int
2008 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
2009                                       const char *extern_type_name,
2010                                       const char *name,
2011                                       const char *args)
2012 {
2013         struct extern_type *type;
2014         struct extern_obj *obj;
2015         void *obj_handle;
2016
2017         CHECK(p, EINVAL);
2018
2019         CHECK_NAME(extern_type_name, EINVAL);
2020         type = extern_type_find(p, extern_type_name);
2021         CHECK(type, EINVAL);
2022
2023         CHECK_NAME(name, EINVAL);
2024         CHECK(!extern_obj_find(p, name), EEXIST);
2025
2026         /* Node allocation. */
2027         obj = calloc(1, sizeof(struct extern_obj));
2028         CHECK(obj, ENOMEM);
2029
2030         /* Object construction. */
2031         obj_handle = type->constructor(args);
2032         if (!obj_handle) {
2033                 free(obj);
2034                 CHECK(0, ENODEV);
2035         }
2036
2037         /* Node initialization. */
2038         strcpy(obj->name, name);
2039         obj->type = type;
2040         obj->obj = obj_handle;
2041         obj->struct_id = p->n_structs;
2042         obj->id = p->n_extern_objs;
2043
2044         /* Node add to tailq. */
2045         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
2046         p->n_extern_objs++;
2047         p->n_structs++;
2048
2049         return 0;
2050 }
2051
2052 static int
2053 extern_obj_build(struct rte_swx_pipeline *p)
2054 {
2055         uint32_t i;
2056
2057         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2058                 struct thread *t = &p->threads[i];
2059                 struct extern_obj *obj;
2060
2061                 t->extern_objs = calloc(p->n_extern_objs,
2062                                         sizeof(struct extern_obj_runtime));
2063                 CHECK(t->extern_objs, ENOMEM);
2064
2065                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
2066                         struct extern_obj_runtime *r =
2067                                 &t->extern_objs[obj->id];
2068                         struct extern_type_member_func *func;
2069                         uint32_t mailbox_size =
2070                                 obj->type->mailbox_struct_type->n_bits / 8;
2071
2072                         r->obj = obj->obj;
2073
2074                         r->mailbox = calloc(1, mailbox_size);
2075                         CHECK(r->mailbox, ENOMEM);
2076
2077                         TAILQ_FOREACH(func, &obj->type->funcs, node)
2078                                 r->funcs[func->id] = func->func;
2079
2080                         t->structs[obj->struct_id] = r->mailbox;
2081                 }
2082         }
2083
2084         return 0;
2085 }
2086
2087 static void
2088 extern_obj_build_free(struct rte_swx_pipeline *p)
2089 {
2090         uint32_t i;
2091
2092         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2093                 struct thread *t = &p->threads[i];
2094                 uint32_t j;
2095
2096                 if (!t->extern_objs)
2097                         continue;
2098
2099                 for (j = 0; j < p->n_extern_objs; j++) {
2100                         struct extern_obj_runtime *r = &t->extern_objs[j];
2101
2102                         free(r->mailbox);
2103                 }
2104
2105                 free(t->extern_objs);
2106                 t->extern_objs = NULL;
2107         }
2108 }
2109
2110 static void
2111 extern_obj_free(struct rte_swx_pipeline *p)
2112 {
2113         extern_obj_build_free(p);
2114
2115         /* Extern objects. */
2116         for ( ; ; ) {
2117                 struct extern_obj *elem;
2118
2119                 elem = TAILQ_FIRST(&p->extern_objs);
2120                 if (!elem)
2121                         break;
2122
2123                 TAILQ_REMOVE(&p->extern_objs, elem, node);
2124                 if (elem->obj)
2125                         elem->type->destructor(elem->obj);
2126                 free(elem);
2127         }
2128
2129         /* Extern types. */
2130         for ( ; ; ) {
2131                 struct extern_type *elem;
2132
2133                 elem = TAILQ_FIRST(&p->extern_types);
2134                 if (!elem)
2135                         break;
2136
2137                 TAILQ_REMOVE(&p->extern_types, elem, node);
2138
2139                 for ( ; ; ) {
2140                         struct extern_type_member_func *func;
2141
2142                         func = TAILQ_FIRST(&elem->funcs);
2143                         if (!func)
2144                                 break;
2145
2146                         TAILQ_REMOVE(&elem->funcs, func, node);
2147                         free(func);
2148                 }
2149
2150                 free(elem);
2151         }
2152 }
2153
2154 /*
2155  * Extern function.
2156  */
2157 static struct extern_func *
2158 extern_func_find(struct rte_swx_pipeline *p, const char *name)
2159 {
2160         struct extern_func *elem;
2161
2162         TAILQ_FOREACH(elem, &p->extern_funcs, node)
2163                 if (strcmp(elem->name, name) == 0)
2164                         return elem;
2165
2166         return NULL;
2167 }
2168
2169 static struct extern_func *
2170 extern_func_parse(struct rte_swx_pipeline *p,
2171                   const char *name)
2172 {
2173         if (name[0] != 'f' || name[1] != '.')
2174                 return NULL;
2175
2176         return extern_func_find(p, &name[2]);
2177 }
2178
2179 static struct field *
2180 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
2181                                 const char *name,
2182                                 struct extern_func **function)
2183 {
2184         struct extern_func *func;
2185         struct field *f;
2186         char *func_name, *field_name;
2187
2188         if ((name[0] != 'f') || (name[1] != '.'))
2189                 return NULL;
2190
2191         func_name = strdup(&name[2]);
2192         if (!func_name)
2193                 return NULL;
2194
2195         field_name = strchr(func_name, '.');
2196         if (!field_name) {
2197                 free(func_name);
2198                 return NULL;
2199         }
2200
2201         *field_name = 0;
2202         field_name++;
2203
2204         func = extern_func_find(p, func_name);
2205         if (!func) {
2206                 free(func_name);
2207                 return NULL;
2208         }
2209
2210         f = struct_type_field_find(func->mailbox_struct_type, field_name);
2211         if (!f) {
2212                 free(func_name);
2213                 return NULL;
2214         }
2215
2216         if (function)
2217                 *function = func;
2218
2219         free(func_name);
2220         return f;
2221 }
2222
2223 int
2224 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
2225                                       const char *name,
2226                                       const char *mailbox_struct_type_name,
2227                                       rte_swx_extern_func_t func)
2228 {
2229         struct extern_func *f;
2230         struct struct_type *mailbox_struct_type;
2231
2232         CHECK(p, EINVAL);
2233
2234         CHECK_NAME(name, EINVAL);
2235         CHECK(!extern_func_find(p, name), EEXIST);
2236
2237         CHECK_NAME(mailbox_struct_type_name, EINVAL);
2238         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2239         CHECK(mailbox_struct_type, EINVAL);
2240
2241         CHECK(func, EINVAL);
2242
2243         /* Node allocation. */
2244         f = calloc(1, sizeof(struct extern_func));
2245         CHECK(func, ENOMEM);
2246
2247         /* Node initialization. */
2248         strcpy(f->name, name);
2249         f->mailbox_struct_type = mailbox_struct_type;
2250         f->func = func;
2251         f->struct_id = p->n_structs;
2252         f->id = p->n_extern_funcs;
2253
2254         /* Node add to tailq. */
2255         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
2256         p->n_extern_funcs++;
2257         p->n_structs++;
2258
2259         return 0;
2260 }
2261
2262 static int
2263 extern_func_build(struct rte_swx_pipeline *p)
2264 {
2265         uint32_t i;
2266
2267         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2268                 struct thread *t = &p->threads[i];
2269                 struct extern_func *func;
2270
2271                 /* Memory allocation. */
2272                 t->extern_funcs = calloc(p->n_extern_funcs,
2273                                          sizeof(struct extern_func_runtime));
2274                 CHECK(t->extern_funcs, ENOMEM);
2275
2276                 /* Extern function. */
2277                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
2278                         struct extern_func_runtime *r =
2279                                 &t->extern_funcs[func->id];
2280                         uint32_t mailbox_size =
2281                                 func->mailbox_struct_type->n_bits / 8;
2282
2283                         r->func = func->func;
2284
2285                         r->mailbox = calloc(1, mailbox_size);
2286                         CHECK(r->mailbox, ENOMEM);
2287
2288                         t->structs[func->struct_id] = r->mailbox;
2289                 }
2290         }
2291
2292         return 0;
2293 }
2294
2295 static void
2296 extern_func_build_free(struct rte_swx_pipeline *p)
2297 {
2298         uint32_t i;
2299
2300         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2301                 struct thread *t = &p->threads[i];
2302                 uint32_t j;
2303
2304                 if (!t->extern_funcs)
2305                         continue;
2306
2307                 for (j = 0; j < p->n_extern_funcs; j++) {
2308                         struct extern_func_runtime *r = &t->extern_funcs[j];
2309
2310                         free(r->mailbox);
2311                 }
2312
2313                 free(t->extern_funcs);
2314                 t->extern_funcs = NULL;
2315         }
2316 }
2317
2318 static void
2319 extern_func_free(struct rte_swx_pipeline *p)
2320 {
2321         extern_func_build_free(p);
2322
2323         for ( ; ; ) {
2324                 struct extern_func *elem;
2325
2326                 elem = TAILQ_FIRST(&p->extern_funcs);
2327                 if (!elem)
2328                         break;
2329
2330                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
2331                 free(elem);
2332         }
2333 }
2334
2335 /*
2336  * Header.
2337  */
2338 static struct header *
2339 header_find(struct rte_swx_pipeline *p, const char *name)
2340 {
2341         struct header *elem;
2342
2343         TAILQ_FOREACH(elem, &p->headers, node)
2344                 if (strcmp(elem->name, name) == 0)
2345                         return elem;
2346
2347         return NULL;
2348 }
2349
2350 static struct header *
2351 header_parse(struct rte_swx_pipeline *p,
2352              const char *name)
2353 {
2354         if (name[0] != 'h' || name[1] != '.')
2355                 return NULL;
2356
2357         return header_find(p, &name[2]);
2358 }
2359
2360 static struct field *
2361 header_field_parse(struct rte_swx_pipeline *p,
2362                    const char *name,
2363                    struct header **header)
2364 {
2365         struct header *h;
2366         struct field *f;
2367         char *header_name, *field_name;
2368
2369         if ((name[0] != 'h') || (name[1] != '.'))
2370                 return NULL;
2371
2372         header_name = strdup(&name[2]);
2373         if (!header_name)
2374                 return NULL;
2375
2376         field_name = strchr(header_name, '.');
2377         if (!field_name) {
2378                 free(header_name);
2379                 return NULL;
2380         }
2381
2382         *field_name = 0;
2383         field_name++;
2384
2385         h = header_find(p, header_name);
2386         if (!h) {
2387                 free(header_name);
2388                 return NULL;
2389         }
2390
2391         f = struct_type_field_find(h->st, field_name);
2392         if (!f) {
2393                 free(header_name);
2394                 return NULL;
2395         }
2396
2397         if (header)
2398                 *header = h;
2399
2400         free(header_name);
2401         return f;
2402 }
2403
2404 int
2405 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2406                                         const char *name,
2407                                         const char *struct_type_name)
2408 {
2409         struct struct_type *st;
2410         struct header *h;
2411         size_t n_headers_max;
2412
2413         CHECK(p, EINVAL);
2414         CHECK_NAME(name, EINVAL);
2415         CHECK_NAME(struct_type_name, EINVAL);
2416
2417         CHECK(!header_find(p, name), EEXIST);
2418
2419         st = struct_type_find(p, struct_type_name);
2420         CHECK(st, EINVAL);
2421
2422         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2423         CHECK(p->n_headers < n_headers_max, ENOSPC);
2424
2425         /* Node allocation. */
2426         h = calloc(1, sizeof(struct header));
2427         CHECK(h, ENOMEM);
2428
2429         /* Node initialization. */
2430         strcpy(h->name, name);
2431         h->st = st;
2432         h->struct_id = p->n_structs;
2433         h->id = p->n_headers;
2434
2435         /* Node add to tailq. */
2436         TAILQ_INSERT_TAIL(&p->headers, h, node);
2437         p->n_headers++;
2438         p->n_structs++;
2439
2440         return 0;
2441 }
2442
2443 static int
2444 header_build(struct rte_swx_pipeline *p)
2445 {
2446         struct header *h;
2447         uint32_t n_bytes = 0, i;
2448
2449         TAILQ_FOREACH(h, &p->headers, node) {
2450                 n_bytes += h->st->n_bits / 8;
2451         }
2452
2453         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2454                 struct thread *t = &p->threads[i];
2455                 uint32_t offset = 0;
2456
2457                 t->headers = calloc(p->n_headers,
2458                                     sizeof(struct header_runtime));
2459                 CHECK(t->headers, ENOMEM);
2460
2461                 t->headers_out = calloc(p->n_headers,
2462                                         sizeof(struct header_out_runtime));
2463                 CHECK(t->headers_out, ENOMEM);
2464
2465                 t->header_storage = calloc(1, n_bytes);
2466                 CHECK(t->header_storage, ENOMEM);
2467
2468                 t->header_out_storage = calloc(1, n_bytes);
2469                 CHECK(t->header_out_storage, ENOMEM);
2470
2471                 TAILQ_FOREACH(h, &p->headers, node) {
2472                         uint8_t *header_storage;
2473
2474                         header_storage = &t->header_storage[offset];
2475                         offset += h->st->n_bits / 8;
2476
2477                         t->headers[h->id].ptr0 = header_storage;
2478                         t->structs[h->struct_id] = header_storage;
2479                 }
2480         }
2481
2482         return 0;
2483 }
2484
2485 static void
2486 header_build_free(struct rte_swx_pipeline *p)
2487 {
2488         uint32_t i;
2489
2490         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2491                 struct thread *t = &p->threads[i];
2492
2493                 free(t->headers_out);
2494                 t->headers_out = NULL;
2495
2496                 free(t->headers);
2497                 t->headers = NULL;
2498
2499                 free(t->header_out_storage);
2500                 t->header_out_storage = NULL;
2501
2502                 free(t->header_storage);
2503                 t->header_storage = NULL;
2504         }
2505 }
2506
2507 static void
2508 header_free(struct rte_swx_pipeline *p)
2509 {
2510         header_build_free(p);
2511
2512         for ( ; ; ) {
2513                 struct header *elem;
2514
2515                 elem = TAILQ_FIRST(&p->headers);
2516                 if (!elem)
2517                         break;
2518
2519                 TAILQ_REMOVE(&p->headers, elem, node);
2520                 free(elem);
2521         }
2522 }
2523
2524 /*
2525  * Meta-data.
2526  */
2527 static struct field *
2528 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2529 {
2530         if (!p->metadata_st)
2531                 return NULL;
2532
2533         if (name[0] != 'm' || name[1] != '.')
2534                 return NULL;
2535
2536         return struct_type_field_find(p->metadata_st, &name[2]);
2537 }
2538
2539 int
2540 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2541                                           const char *struct_type_name)
2542 {
2543         struct struct_type *st = NULL;
2544
2545         CHECK(p, EINVAL);
2546
2547         CHECK_NAME(struct_type_name, EINVAL);
2548         st  = struct_type_find(p, struct_type_name);
2549         CHECK(st, EINVAL);
2550         CHECK(!p->metadata_st, EINVAL);
2551
2552         p->metadata_st = st;
2553         p->metadata_struct_id = p->n_structs;
2554
2555         p->n_structs++;
2556
2557         return 0;
2558 }
2559
2560 static int
2561 metadata_build(struct rte_swx_pipeline *p)
2562 {
2563         uint32_t n_bytes = p->metadata_st->n_bits / 8;
2564         uint32_t i;
2565
2566         /* Thread-level initialization. */
2567         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2568                 struct thread *t = &p->threads[i];
2569                 uint8_t *metadata;
2570
2571                 metadata = calloc(1, n_bytes);
2572                 CHECK(metadata, ENOMEM);
2573
2574                 t->metadata = metadata;
2575                 t->structs[p->metadata_struct_id] = metadata;
2576         }
2577
2578         return 0;
2579 }
2580
2581 static void
2582 metadata_build_free(struct rte_swx_pipeline *p)
2583 {
2584         uint32_t i;
2585
2586         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2587                 struct thread *t = &p->threads[i];
2588
2589                 free(t->metadata);
2590                 t->metadata = NULL;
2591         }
2592 }
2593
2594 static void
2595 metadata_free(struct rte_swx_pipeline *p)
2596 {
2597         metadata_build_free(p);
2598 }
2599
2600 /*
2601  * Instruction.
2602  */
2603 static int
2604 instruction_is_tx(enum instruction_type type)
2605 {
2606         switch (type) {
2607         case INSTR_TX:
2608         case INSTR_TX_I:
2609                 return 1;
2610
2611         default:
2612                 return 0;
2613         }
2614 }
2615
2616 static int
2617 instruction_is_jmp(struct instruction *instr)
2618 {
2619         switch (instr->type) {
2620         case INSTR_JMP:
2621         case INSTR_JMP_VALID:
2622         case INSTR_JMP_INVALID:
2623         case INSTR_JMP_HIT:
2624         case INSTR_JMP_MISS:
2625         case INSTR_JMP_ACTION_HIT:
2626         case INSTR_JMP_ACTION_MISS:
2627         case INSTR_JMP_EQ:
2628         case INSTR_JMP_EQ_MH:
2629         case INSTR_JMP_EQ_HM:
2630         case INSTR_JMP_EQ_HH:
2631         case INSTR_JMP_EQ_I:
2632         case INSTR_JMP_NEQ:
2633         case INSTR_JMP_NEQ_MH:
2634         case INSTR_JMP_NEQ_HM:
2635         case INSTR_JMP_NEQ_HH:
2636         case INSTR_JMP_NEQ_I:
2637         case INSTR_JMP_LT:
2638         case INSTR_JMP_LT_MH:
2639         case INSTR_JMP_LT_HM:
2640         case INSTR_JMP_LT_HH:
2641         case INSTR_JMP_LT_MI:
2642         case INSTR_JMP_LT_HI:
2643         case INSTR_JMP_GT:
2644         case INSTR_JMP_GT_MH:
2645         case INSTR_JMP_GT_HM:
2646         case INSTR_JMP_GT_HH:
2647         case INSTR_JMP_GT_MI:
2648         case INSTR_JMP_GT_HI:
2649                 return 1;
2650
2651         default:
2652                 return 0;
2653         }
2654 }
2655
2656 static struct field *
2657 action_field_parse(struct action *action, const char *name);
2658
2659 static struct field *
2660 struct_field_parse(struct rte_swx_pipeline *p,
2661                    struct action *action,
2662                    const char *name,
2663                    uint32_t *struct_id)
2664 {
2665         struct field *f;
2666
2667         switch (name[0]) {
2668         case 'h':
2669         {
2670                 struct header *header;
2671
2672                 f = header_field_parse(p, name, &header);
2673                 if (!f)
2674                         return NULL;
2675
2676                 *struct_id = header->struct_id;
2677                 return f;
2678         }
2679
2680         case 'm':
2681         {
2682                 f = metadata_field_parse(p, name);
2683                 if (!f)
2684                         return NULL;
2685
2686                 *struct_id = p->metadata_struct_id;
2687                 return f;
2688         }
2689
2690         case 't':
2691         {
2692                 if (!action)
2693                         return NULL;
2694
2695                 f = action_field_parse(action, name);
2696                 if (!f)
2697                         return NULL;
2698
2699                 *struct_id = 0;
2700                 return f;
2701         }
2702
2703         case 'e':
2704         {
2705                 struct extern_obj *obj;
2706
2707                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2708                 if (!f)
2709                         return NULL;
2710
2711                 *struct_id = obj->struct_id;
2712                 return f;
2713         }
2714
2715         case 'f':
2716         {
2717                 struct extern_func *func;
2718
2719                 f = extern_func_mailbox_field_parse(p, name, &func);
2720                 if (!f)
2721                         return NULL;
2722
2723                 *struct_id = func->struct_id;
2724                 return f;
2725         }
2726
2727         default:
2728                 return NULL;
2729         }
2730 }
2731
2732 static inline void
2733 pipeline_port_inc(struct rte_swx_pipeline *p)
2734 {
2735         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2736 }
2737
2738 static inline void
2739 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2740 {
2741         t->ip = p->instructions;
2742 }
2743
2744 static inline void
2745 thread_ip_set(struct thread *t, struct instruction *ip)
2746 {
2747         t->ip = ip;
2748 }
2749
2750 static inline void
2751 thread_ip_action_call(struct rte_swx_pipeline *p,
2752                       struct thread *t,
2753                       uint32_t action_id)
2754 {
2755         t->ret = t->ip + 1;
2756         t->ip = p->action_instructions[action_id];
2757 }
2758
2759 static inline void
2760 thread_ip_inc(struct rte_swx_pipeline *p);
2761
2762 static inline void
2763 thread_ip_inc(struct rte_swx_pipeline *p)
2764 {
2765         struct thread *t = &p->threads[p->thread_id];
2766
2767         t->ip++;
2768 }
2769
2770 static inline void
2771 thread_ip_inc_cond(struct thread *t, int cond)
2772 {
2773         t->ip += cond;
2774 }
2775
2776 static inline void
2777 thread_yield(struct rte_swx_pipeline *p)
2778 {
2779         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2780 }
2781
2782 static inline void
2783 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2784 {
2785         p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2786 }
2787
2788 /*
2789  * rx.
2790  */
2791 static int
2792 instr_rx_translate(struct rte_swx_pipeline *p,
2793                    struct action *action,
2794                    char **tokens,
2795                    int n_tokens,
2796                    struct instruction *instr,
2797                    struct instruction_data *data __rte_unused)
2798 {
2799         struct field *f;
2800
2801         CHECK(!action, EINVAL);
2802         CHECK(n_tokens == 2, EINVAL);
2803
2804         f = metadata_field_parse(p, tokens[1]);
2805         CHECK(f, EINVAL);
2806
2807         instr->type = INSTR_RX;
2808         instr->io.io.offset = f->offset / 8;
2809         instr->io.io.n_bits = f->n_bits;
2810         return 0;
2811 }
2812
2813 static inline void
2814 instr_rx_exec(struct rte_swx_pipeline *p);
2815
2816 static inline void
2817 instr_rx_exec(struct rte_swx_pipeline *p)
2818 {
2819         struct thread *t = &p->threads[p->thread_id];
2820         struct instruction *ip = t->ip;
2821         struct port_in_runtime *port = &p->in[p->port_id];
2822         struct rte_swx_pkt *pkt = &t->pkt;
2823         int pkt_received;
2824
2825         /* Packet. */
2826         pkt_received = port->pkt_rx(port->obj, pkt);
2827         t->ptr = &pkt->pkt[pkt->offset];
2828         rte_prefetch0(t->ptr);
2829
2830         TRACE("[Thread %2u] rx %s from port %u\n",
2831               p->thread_id,
2832               pkt_received ? "1 pkt" : "0 pkts",
2833               p->port_id);
2834
2835         /* Headers. */
2836         t->valid_headers = 0;
2837         t->n_headers_out = 0;
2838
2839         /* Meta-data. */
2840         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2841
2842         /* Tables. */
2843         t->table_state = p->table_state;
2844
2845         /* Thread. */
2846         pipeline_port_inc(p);
2847         thread_ip_inc_cond(t, pkt_received);
2848         thread_yield(p);
2849 }
2850
2851 /*
2852  * tx.
2853  */
2854 static int
2855 instr_tx_translate(struct rte_swx_pipeline *p,
2856                    struct action *action __rte_unused,
2857                    char **tokens,
2858                    int n_tokens,
2859                    struct instruction *instr,
2860                    struct instruction_data *data __rte_unused)
2861 {
2862         char *port = tokens[1];
2863         struct field *f;
2864         uint32_t port_val;
2865
2866         CHECK(n_tokens == 2, EINVAL);
2867
2868         f = metadata_field_parse(p, port);
2869         if (f) {
2870                 instr->type = INSTR_TX;
2871                 instr->io.io.offset = f->offset / 8;
2872                 instr->io.io.n_bits = f->n_bits;
2873                 return 0;
2874         }
2875
2876         /* TX_I. */
2877         port_val = strtoul(port, &port, 0);
2878         CHECK(!port[0], EINVAL);
2879
2880         instr->type = INSTR_TX_I;
2881         instr->io.io.val = port_val;
2882         return 0;
2883 }
2884
2885 static int
2886 instr_drop_translate(struct rte_swx_pipeline *p,
2887                      struct action *action __rte_unused,
2888                      char **tokens __rte_unused,
2889                      int n_tokens,
2890                      struct instruction *instr,
2891                      struct instruction_data *data __rte_unused)
2892 {
2893         CHECK(n_tokens == 1, EINVAL);
2894
2895         /* TX_I. */
2896         instr->type = INSTR_TX_I;
2897         instr->io.io.val = p->n_ports_out - 1;
2898         return 0;
2899 }
2900
2901 static inline void
2902 emit_handler(struct thread *t)
2903 {
2904         struct header_out_runtime *h0 = &t->headers_out[0];
2905         struct header_out_runtime *h1 = &t->headers_out[1];
2906         uint32_t offset = 0, i;
2907
2908         /* No header change or header decapsulation. */
2909         if ((t->n_headers_out == 1) &&
2910             (h0->ptr + h0->n_bytes == t->ptr)) {
2911                 TRACE("Emit handler: no header change or header decap.\n");
2912
2913                 t->pkt.offset -= h0->n_bytes;
2914                 t->pkt.length += h0->n_bytes;
2915
2916                 return;
2917         }
2918
2919         /* Header encapsulation (optionally, with prior header decasulation). */
2920         if ((t->n_headers_out == 2) &&
2921             (h1->ptr + h1->n_bytes == t->ptr) &&
2922             (h0->ptr == h0->ptr0)) {
2923                 uint32_t offset;
2924
2925                 TRACE("Emit handler: header encapsulation.\n");
2926
2927                 offset = h0->n_bytes + h1->n_bytes;
2928                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2929                 t->pkt.offset -= offset;
2930                 t->pkt.length += offset;
2931
2932                 return;
2933         }
2934
2935         /* Header insertion. */
2936         /* TBD */
2937
2938         /* Header extraction. */
2939         /* TBD */
2940
2941         /* For any other case. */
2942         TRACE("Emit handler: complex case.\n");
2943
2944         for (i = 0; i < t->n_headers_out; i++) {
2945                 struct header_out_runtime *h = &t->headers_out[i];
2946
2947                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2948                 offset += h->n_bytes;
2949         }
2950
2951         if (offset) {
2952                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2953                 t->pkt.offset -= offset;
2954                 t->pkt.length += offset;
2955         }
2956 }
2957
2958 static inline void
2959 instr_tx_exec(struct rte_swx_pipeline *p);
2960
2961 static inline void
2962 instr_tx_exec(struct rte_swx_pipeline *p)
2963 {
2964         struct thread *t = &p->threads[p->thread_id];
2965         struct instruction *ip = t->ip;
2966         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2967         struct port_out_runtime *port = &p->out[port_id];
2968         struct rte_swx_pkt *pkt = &t->pkt;
2969
2970         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2971               p->thread_id,
2972               (uint32_t)port_id);
2973
2974         /* Headers. */
2975         emit_handler(t);
2976
2977         /* Packet. */
2978         port->pkt_tx(port->obj, pkt);
2979
2980         /* Thread. */
2981         thread_ip_reset(p, t);
2982         instr_rx_exec(p);
2983 }
2984
2985 static inline void
2986 instr_tx_i_exec(struct rte_swx_pipeline *p)
2987 {
2988         struct thread *t = &p->threads[p->thread_id];
2989         struct instruction *ip = t->ip;
2990         uint64_t port_id = ip->io.io.val;
2991         struct port_out_runtime *port = &p->out[port_id];
2992         struct rte_swx_pkt *pkt = &t->pkt;
2993
2994         TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
2995               p->thread_id,
2996               (uint32_t)port_id);
2997
2998         /* Headers. */
2999         emit_handler(t);
3000
3001         /* Packet. */
3002         port->pkt_tx(port->obj, pkt);
3003
3004         /* Thread. */
3005         thread_ip_reset(p, t);
3006         instr_rx_exec(p);
3007 }
3008
3009 /*
3010  * extract.
3011  */
3012 static int
3013 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
3014                             struct action *action,
3015                             char **tokens,
3016                             int n_tokens,
3017                             struct instruction *instr,
3018                             struct instruction_data *data __rte_unused)
3019 {
3020         struct header *h;
3021
3022         CHECK(!action, EINVAL);
3023         CHECK(n_tokens == 2, EINVAL);
3024
3025         h = header_parse(p, tokens[1]);
3026         CHECK(h, EINVAL);
3027
3028         instr->type = INSTR_HDR_EXTRACT;
3029         instr->io.hdr.header_id[0] = h->id;
3030         instr->io.hdr.struct_id[0] = h->struct_id;
3031         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3032         return 0;
3033 }
3034
3035 static inline void
3036 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
3037
3038 static inline void
3039 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
3040 {
3041         struct thread *t = &p->threads[p->thread_id];
3042         struct instruction *ip = t->ip;
3043         uint64_t valid_headers = t->valid_headers;
3044         uint8_t *ptr = t->ptr;
3045         uint32_t offset = t->pkt.offset;
3046         uint32_t length = t->pkt.length;
3047         uint32_t i;
3048
3049         for (i = 0; i < n_extract; i++) {
3050                 uint32_t header_id = ip->io.hdr.header_id[i];
3051                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3052                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3053
3054                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
3055                       p->thread_id,
3056                       header_id,
3057                       n_bytes);
3058
3059                 /* Headers. */
3060                 t->structs[struct_id] = ptr;
3061                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3062
3063                 /* Packet. */
3064                 offset += n_bytes;
3065                 length -= n_bytes;
3066                 ptr += n_bytes;
3067         }
3068
3069         /* Headers. */
3070         t->valid_headers = valid_headers;
3071
3072         /* Packet. */
3073         t->pkt.offset = offset;
3074         t->pkt.length = length;
3075         t->ptr = ptr;
3076 }
3077
3078 static inline void
3079 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
3080 {
3081         __instr_hdr_extract_exec(p, 1);
3082
3083         /* Thread. */
3084         thread_ip_inc(p);
3085 }
3086
3087 static inline void
3088 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
3089 {
3090         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3091               p->thread_id);
3092
3093         __instr_hdr_extract_exec(p, 2);
3094
3095         /* Thread. */
3096         thread_ip_inc(p);
3097 }
3098
3099 static inline void
3100 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
3101 {
3102         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3103               p->thread_id);
3104
3105         __instr_hdr_extract_exec(p, 3);
3106
3107         /* Thread. */
3108         thread_ip_inc(p);
3109 }
3110
3111 static inline void
3112 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
3113 {
3114         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3115               p->thread_id);
3116
3117         __instr_hdr_extract_exec(p, 4);
3118
3119         /* Thread. */
3120         thread_ip_inc(p);
3121 }
3122
3123 static inline void
3124 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
3125 {
3126         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3127               p->thread_id);
3128
3129         __instr_hdr_extract_exec(p, 5);
3130
3131         /* Thread. */
3132         thread_ip_inc(p);
3133 }
3134
3135 static inline void
3136 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
3137 {
3138         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3139               p->thread_id);
3140
3141         __instr_hdr_extract_exec(p, 6);
3142
3143         /* Thread. */
3144         thread_ip_inc(p);
3145 }
3146
3147 static inline void
3148 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
3149 {
3150         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3151               p->thread_id);
3152
3153         __instr_hdr_extract_exec(p, 7);
3154
3155         /* Thread. */
3156         thread_ip_inc(p);
3157 }
3158
3159 static inline void
3160 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
3161 {
3162         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3163               p->thread_id);
3164
3165         __instr_hdr_extract_exec(p, 8);
3166
3167         /* Thread. */
3168         thread_ip_inc(p);
3169 }
3170
3171 /*
3172  * emit.
3173  */
3174 static int
3175 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
3176                          struct action *action __rte_unused,
3177                          char **tokens,
3178                          int n_tokens,
3179                          struct instruction *instr,
3180                          struct instruction_data *data __rte_unused)
3181 {
3182         struct header *h;
3183
3184         CHECK(n_tokens == 2, EINVAL);
3185
3186         h = header_parse(p, tokens[1]);
3187         CHECK(h, EINVAL);
3188
3189         instr->type = INSTR_HDR_EMIT;
3190         instr->io.hdr.header_id[0] = h->id;
3191         instr->io.hdr.struct_id[0] = h->struct_id;
3192         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3193         return 0;
3194 }
3195
3196 static inline void
3197 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
3198
3199 static inline void
3200 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
3201 {
3202         struct thread *t = &p->threads[p->thread_id];
3203         struct instruction *ip = t->ip;
3204         uint64_t valid_headers = t->valid_headers;
3205         uint32_t n_headers_out = t->n_headers_out;
3206         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
3207         uint8_t *ho_ptr = NULL;
3208         uint32_t ho_nbytes = 0, first = 1, i;
3209
3210         for (i = 0; i < n_emit; i++) {
3211                 uint32_t header_id = ip->io.hdr.header_id[i];
3212                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3213                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3214
3215                 struct header_runtime *hi = &t->headers[header_id];
3216                 uint8_t *hi_ptr = t->structs[struct_id];
3217
3218                 if (!MASK64_BIT_GET(valid_headers, header_id))
3219                         continue;
3220
3221                 TRACE("[Thread %2u]: emit header %u\n",
3222                       p->thread_id,
3223                       header_id);
3224
3225                 /* Headers. */
3226                 if (first) {
3227                         first = 0;
3228
3229                         if (!t->n_headers_out) {
3230                                 ho = &t->headers_out[0];
3231
3232                                 ho->ptr0 = hi->ptr0;
3233                                 ho->ptr = hi_ptr;
3234
3235                                 ho_ptr = hi_ptr;
3236                                 ho_nbytes = n_bytes;
3237
3238                                 n_headers_out = 1;
3239
3240                                 continue;
3241                         } else {
3242                                 ho_ptr = ho->ptr;
3243                                 ho_nbytes = ho->n_bytes;
3244                         }
3245                 }
3246
3247                 if (ho_ptr + ho_nbytes == hi_ptr) {
3248                         ho_nbytes += n_bytes;
3249                 } else {
3250                         ho->n_bytes = ho_nbytes;
3251
3252                         ho++;
3253                         ho->ptr0 = hi->ptr0;
3254                         ho->ptr = hi_ptr;
3255
3256                         ho_ptr = hi_ptr;
3257                         ho_nbytes = n_bytes;
3258
3259                         n_headers_out++;
3260                 }
3261         }
3262
3263         ho->n_bytes = ho_nbytes;
3264         t->n_headers_out = n_headers_out;
3265 }
3266
3267 static inline void
3268 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
3269 {
3270         __instr_hdr_emit_exec(p, 1);
3271
3272         /* Thread. */
3273         thread_ip_inc(p);
3274 }
3275
3276 static inline void
3277 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
3278 {
3279         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3280               p->thread_id);
3281
3282         __instr_hdr_emit_exec(p, 1);
3283         instr_tx_exec(p);
3284 }
3285
3286 static inline void
3287 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
3288 {
3289         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3290               p->thread_id);
3291
3292         __instr_hdr_emit_exec(p, 2);
3293         instr_tx_exec(p);
3294 }
3295
3296 static inline void
3297 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
3298 {
3299         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3300               p->thread_id);
3301
3302         __instr_hdr_emit_exec(p, 3);
3303         instr_tx_exec(p);
3304 }
3305
3306 static inline void
3307 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
3308 {
3309         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3310               p->thread_id);
3311
3312         __instr_hdr_emit_exec(p, 4);
3313         instr_tx_exec(p);
3314 }
3315
3316 static inline void
3317 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
3318 {
3319         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3320               p->thread_id);
3321
3322         __instr_hdr_emit_exec(p, 5);
3323         instr_tx_exec(p);
3324 }
3325
3326 static inline void
3327 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
3328 {
3329         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3330               p->thread_id);
3331
3332         __instr_hdr_emit_exec(p, 6);
3333         instr_tx_exec(p);
3334 }
3335
3336 static inline void
3337 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
3338 {
3339         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3340               p->thread_id);
3341
3342         __instr_hdr_emit_exec(p, 7);
3343         instr_tx_exec(p);
3344 }
3345
3346 static inline void
3347 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
3348 {
3349         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
3350               p->thread_id);
3351
3352         __instr_hdr_emit_exec(p, 8);
3353         instr_tx_exec(p);
3354 }
3355
3356 /*
3357  * validate.
3358  */
3359 static int
3360 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
3361                              struct action *action __rte_unused,
3362                              char **tokens,
3363                              int n_tokens,
3364                              struct instruction *instr,
3365                              struct instruction_data *data __rte_unused)
3366 {
3367         struct header *h;
3368
3369         CHECK(n_tokens == 2, EINVAL);
3370
3371         h = header_parse(p, tokens[1]);
3372         CHECK(h, EINVAL);
3373
3374         instr->type = INSTR_HDR_VALIDATE;
3375         instr->valid.header_id = h->id;
3376         return 0;
3377 }
3378
3379 static inline void
3380 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
3381 {
3382         struct thread *t = &p->threads[p->thread_id];
3383         struct instruction *ip = t->ip;
3384         uint32_t header_id = ip->valid.header_id;
3385
3386         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
3387
3388         /* Headers. */
3389         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
3390
3391         /* Thread. */
3392         thread_ip_inc(p);
3393 }
3394
3395 /*
3396  * invalidate.
3397  */
3398 static int
3399 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
3400                                struct action *action __rte_unused,
3401                                char **tokens,
3402                                int n_tokens,
3403                                struct instruction *instr,
3404                                struct instruction_data *data __rte_unused)
3405 {
3406         struct header *h;
3407
3408         CHECK(n_tokens == 2, EINVAL);
3409
3410         h = header_parse(p, tokens[1]);
3411         CHECK(h, EINVAL);
3412
3413         instr->type = INSTR_HDR_INVALIDATE;
3414         instr->valid.header_id = h->id;
3415         return 0;
3416 }
3417
3418 static inline void
3419 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3420 {
3421         struct thread *t = &p->threads[p->thread_id];
3422         struct instruction *ip = t->ip;
3423         uint32_t header_id = ip->valid.header_id;
3424
3425         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3426
3427         /* Headers. */
3428         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3429
3430         /* Thread. */
3431         thread_ip_inc(p);
3432 }
3433
3434 /*
3435  * table.
3436  */
3437 static struct table *
3438 table_find(struct rte_swx_pipeline *p, const char *name);
3439
3440 static int
3441 instr_table_translate(struct rte_swx_pipeline *p,
3442                       struct action *action,
3443                       char **tokens,
3444                       int n_tokens,
3445                       struct instruction *instr,
3446                       struct instruction_data *data __rte_unused)
3447 {
3448         struct table *t;
3449
3450         CHECK(!action, EINVAL);
3451         CHECK(n_tokens == 2, EINVAL);
3452
3453         t = table_find(p, tokens[1]);
3454         CHECK(t, EINVAL);
3455
3456         instr->type = INSTR_TABLE;
3457         instr->table.table_id = t->id;
3458         return 0;
3459 }
3460
3461 static inline void
3462 instr_table_exec(struct rte_swx_pipeline *p)
3463 {
3464         struct thread *t = &p->threads[p->thread_id];
3465         struct instruction *ip = t->ip;
3466         uint32_t table_id = ip->table.table_id;
3467         struct rte_swx_table_state *ts = &t->table_state[table_id];
3468         struct table_runtime *table = &t->tables[table_id];
3469         struct table_statistics *stats = &p->table_stats[table_id];
3470         uint64_t action_id, n_pkts_hit, n_pkts_action;
3471         uint8_t *action_data;
3472         int done, hit;
3473
3474         /* Table. */
3475         done = table->func(ts->obj,
3476                            table->mailbox,
3477                            table->key,
3478                            &action_id,
3479                            &action_data,
3480                            &hit);
3481         if (!done) {
3482                 /* Thread. */
3483                 TRACE("[Thread %2u] table %u (not finalized)\n",
3484                       p->thread_id,
3485                       table_id);
3486
3487                 thread_yield(p);
3488                 return;
3489         }
3490
3491         action_id = hit ? action_id : ts->default_action_id;
3492         action_data = hit ? action_data : ts->default_action_data;
3493         n_pkts_hit = stats->n_pkts_hit[hit];
3494         n_pkts_action = stats->n_pkts_action[action_id];
3495
3496         TRACE("[Thread %2u] table %u (%s, action %u)\n",
3497               p->thread_id,
3498               table_id,
3499               hit ? "hit" : "miss",
3500               (uint32_t)action_id);
3501
3502         t->action_id = action_id;
3503         t->structs[0] = action_data;
3504         t->hit = hit;
3505         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
3506         stats->n_pkts_action[action_id] = n_pkts_action + 1;
3507
3508         /* Thread. */
3509         thread_ip_action_call(p, t, action_id);
3510 }
3511
3512 /*
3513  * extern.
3514  */
3515 static int
3516 instr_extern_translate(struct rte_swx_pipeline *p,
3517                        struct action *action __rte_unused,
3518                        char **tokens,
3519                        int n_tokens,
3520                        struct instruction *instr,
3521                        struct instruction_data *data __rte_unused)
3522 {
3523         char *token = tokens[1];
3524
3525         CHECK(n_tokens == 2, EINVAL);
3526
3527         if (token[0] == 'e') {
3528                 struct extern_obj *obj;
3529                 struct extern_type_member_func *func;
3530
3531                 func = extern_obj_member_func_parse(p, token, &obj);
3532                 CHECK(func, EINVAL);
3533
3534                 instr->type = INSTR_EXTERN_OBJ;
3535                 instr->ext_obj.ext_obj_id = obj->id;
3536                 instr->ext_obj.func_id = func->id;
3537
3538                 return 0;
3539         }
3540
3541         if (token[0] == 'f') {
3542                 struct extern_func *func;
3543
3544                 func = extern_func_parse(p, token);
3545                 CHECK(func, EINVAL);
3546
3547                 instr->type = INSTR_EXTERN_FUNC;
3548                 instr->ext_func.ext_func_id = func->id;
3549
3550                 return 0;
3551         }
3552
3553         CHECK(0, EINVAL);
3554 }
3555
3556 static inline void
3557 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3558 {
3559         struct thread *t = &p->threads[p->thread_id];
3560         struct instruction *ip = t->ip;
3561         uint32_t obj_id = ip->ext_obj.ext_obj_id;
3562         uint32_t func_id = ip->ext_obj.func_id;
3563         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3564         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3565
3566         TRACE("[Thread %2u] extern obj %u member func %u\n",
3567               p->thread_id,
3568               obj_id,
3569               func_id);
3570
3571         /* Extern object member function execute. */
3572         uint32_t done = func(obj->obj, obj->mailbox);
3573
3574         /* Thread. */
3575         thread_ip_inc_cond(t, done);
3576         thread_yield_cond(p, done ^ 1);
3577 }
3578
3579 static inline void
3580 instr_extern_func_exec(struct rte_swx_pipeline *p)
3581 {
3582         struct thread *t = &p->threads[p->thread_id];
3583         struct instruction *ip = t->ip;
3584         uint32_t ext_func_id = ip->ext_func.ext_func_id;
3585         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3586         rte_swx_extern_func_t func = ext_func->func;
3587
3588         TRACE("[Thread %2u] extern func %u\n",
3589               p->thread_id,
3590               ext_func_id);
3591
3592         /* Extern function execute. */
3593         uint32_t done = func(ext_func->mailbox);
3594
3595         /* Thread. */
3596         thread_ip_inc_cond(t, done);
3597         thread_yield_cond(p, done ^ 1);
3598 }
3599
3600 /*
3601  * mov.
3602  */
3603 static int
3604 instr_mov_translate(struct rte_swx_pipeline *p,
3605                     struct action *action,
3606                     char **tokens,
3607                     int n_tokens,
3608                     struct instruction *instr,
3609                     struct instruction_data *data __rte_unused)
3610 {
3611         char *dst = tokens[1], *src = tokens[2];
3612         struct field *fdst, *fsrc;
3613         uint64_t src_val;
3614         uint32_t dst_struct_id, src_struct_id;
3615
3616         CHECK(n_tokens == 3, EINVAL);
3617
3618         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3619         CHECK(fdst, EINVAL);
3620
3621         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
3622         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3623         if (fsrc) {
3624                 instr->type = INSTR_MOV;
3625                 if (dst[0] != 'h' && src[0] == 'h')
3626                         instr->type = INSTR_MOV_MH;
3627                 if (dst[0] == 'h' && src[0] != 'h')
3628                         instr->type = INSTR_MOV_HM;
3629                 if (dst[0] == 'h' && src[0] == 'h')
3630                         instr->type = INSTR_MOV_HH;
3631
3632                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3633                 instr->mov.dst.n_bits = fdst->n_bits;
3634                 instr->mov.dst.offset = fdst->offset / 8;
3635                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3636                 instr->mov.src.n_bits = fsrc->n_bits;
3637                 instr->mov.src.offset = fsrc->offset / 8;
3638                 return 0;
3639         }
3640
3641         /* MOV_I. */
3642         src_val = strtoull(src, &src, 0);
3643         CHECK(!src[0], EINVAL);
3644
3645         if (dst[0] == 'h')
3646                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3647
3648         instr->type = INSTR_MOV_I;
3649         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3650         instr->mov.dst.n_bits = fdst->n_bits;
3651         instr->mov.dst.offset = fdst->offset / 8;
3652         instr->mov.src_val = src_val;
3653         return 0;
3654 }
3655
3656 static inline void
3657 instr_mov_exec(struct rte_swx_pipeline *p)
3658 {
3659         struct thread *t = &p->threads[p->thread_id];
3660         struct instruction *ip = t->ip;
3661
3662         TRACE("[Thread %2u] mov\n",
3663               p->thread_id);
3664
3665         MOV(t, ip);
3666
3667         /* Thread. */
3668         thread_ip_inc(p);
3669 }
3670
3671 static inline void
3672 instr_mov_mh_exec(struct rte_swx_pipeline *p)
3673 {
3674         struct thread *t = &p->threads[p->thread_id];
3675         struct instruction *ip = t->ip;
3676
3677         TRACE("[Thread %2u] mov (mh)\n",
3678               p->thread_id);
3679
3680         MOV_MH(t, ip);
3681
3682         /* Thread. */
3683         thread_ip_inc(p);
3684 }
3685
3686 static inline void
3687 instr_mov_hm_exec(struct rte_swx_pipeline *p)
3688 {
3689         struct thread *t = &p->threads[p->thread_id];
3690         struct instruction *ip = t->ip;
3691
3692         TRACE("[Thread %2u] mov (hm)\n",
3693               p->thread_id);
3694
3695         MOV_HM(t, ip);
3696
3697         /* Thread. */
3698         thread_ip_inc(p);
3699 }
3700
3701 static inline void
3702 instr_mov_hh_exec(struct rte_swx_pipeline *p)
3703 {
3704         struct thread *t = &p->threads[p->thread_id];
3705         struct instruction *ip = t->ip;
3706
3707         TRACE("[Thread %2u] mov (hh)\n",
3708               p->thread_id);
3709
3710         MOV_HH(t, ip);
3711
3712         /* Thread. */
3713         thread_ip_inc(p);
3714 }
3715
3716 static inline void
3717 instr_mov_i_exec(struct rte_swx_pipeline *p)
3718 {
3719         struct thread *t = &p->threads[p->thread_id];
3720         struct instruction *ip = t->ip;
3721
3722         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
3723               p->thread_id,
3724               ip->mov.src_val);
3725
3726         MOV_I(t, ip);
3727
3728         /* Thread. */
3729         thread_ip_inc(p);
3730 }
3731
3732 /*
3733  * dma.
3734  */
3735 static int
3736 instr_dma_translate(struct rte_swx_pipeline *p,
3737                     struct action *action,
3738                     char **tokens,
3739                     int n_tokens,
3740                     struct instruction *instr,
3741                     struct instruction_data *data __rte_unused)
3742 {
3743         char *dst = tokens[1];
3744         char *src = tokens[2];
3745         struct header *h;
3746         struct field *tf;
3747
3748         CHECK(action, EINVAL);
3749         CHECK(n_tokens == 3, EINVAL);
3750
3751         h = header_parse(p, dst);
3752         CHECK(h, EINVAL);
3753
3754         tf = action_field_parse(action, src);
3755         CHECK(tf, EINVAL);
3756
3757         instr->type = INSTR_DMA_HT;
3758         instr->dma.dst.header_id[0] = h->id;
3759         instr->dma.dst.struct_id[0] = h->struct_id;
3760         instr->dma.n_bytes[0] = h->st->n_bits / 8;
3761         instr->dma.src.offset[0] = tf->offset / 8;
3762
3763         return 0;
3764 }
3765
3766 static inline void
3767 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3768
3769 static inline void
3770 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3771 {
3772         struct thread *t = &p->threads[p->thread_id];
3773         struct instruction *ip = t->ip;
3774         uint8_t *action_data = t->structs[0];
3775         uint64_t valid_headers = t->valid_headers;
3776         uint32_t i;
3777
3778         for (i = 0; i < n_dma; i++) {
3779                 uint32_t header_id = ip->dma.dst.header_id[i];
3780                 uint32_t struct_id = ip->dma.dst.struct_id[i];
3781                 uint32_t offset = ip->dma.src.offset[i];
3782                 uint32_t n_bytes = ip->dma.n_bytes[i];
3783
3784                 struct header_runtime *h = &t->headers[header_id];
3785                 uint8_t *h_ptr0 = h->ptr0;
3786                 uint8_t *h_ptr = t->structs[struct_id];
3787
3788                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3789                         h_ptr : h_ptr0;
3790                 void *src = &action_data[offset];
3791
3792                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3793
3794                 /* Headers. */
3795                 memcpy(dst, src, n_bytes);
3796                 t->structs[struct_id] = dst;
3797                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3798         }
3799
3800         t->valid_headers = valid_headers;
3801 }
3802
3803 static inline void
3804 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3805 {
3806         __instr_dma_ht_exec(p, 1);
3807
3808         /* Thread. */
3809         thread_ip_inc(p);
3810 }
3811
3812 static inline void
3813 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3814 {
3815         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3816               p->thread_id);
3817
3818         __instr_dma_ht_exec(p, 2);
3819
3820         /* Thread. */
3821         thread_ip_inc(p);
3822 }
3823
3824 static inline void
3825 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3826 {
3827         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3828               p->thread_id);
3829
3830         __instr_dma_ht_exec(p, 3);
3831
3832         /* Thread. */
3833         thread_ip_inc(p);
3834 }
3835
3836 static inline void
3837 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3838 {
3839         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3840               p->thread_id);
3841
3842         __instr_dma_ht_exec(p, 4);
3843
3844         /* Thread. */
3845         thread_ip_inc(p);
3846 }
3847
3848 static inline void
3849 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3850 {
3851         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3852               p->thread_id);
3853
3854         __instr_dma_ht_exec(p, 5);
3855
3856         /* Thread. */
3857         thread_ip_inc(p);
3858 }
3859
3860 static inline void
3861 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3862 {
3863         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3864               p->thread_id);
3865
3866         __instr_dma_ht_exec(p, 6);
3867
3868         /* Thread. */
3869         thread_ip_inc(p);
3870 }
3871
3872 static inline void
3873 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3874 {
3875         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3876               p->thread_id);
3877
3878         __instr_dma_ht_exec(p, 7);
3879
3880         /* Thread. */
3881         thread_ip_inc(p);
3882 }
3883
3884 static inline void
3885 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3886 {
3887         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3888               p->thread_id);
3889
3890         __instr_dma_ht_exec(p, 8);
3891
3892         /* Thread. */
3893         thread_ip_inc(p);
3894 }
3895
3896 /*
3897  * alu.
3898  */
3899 static int
3900 instr_alu_add_translate(struct rte_swx_pipeline *p,
3901                         struct action *action,
3902                         char **tokens,
3903                         int n_tokens,
3904                         struct instruction *instr,
3905                         struct instruction_data *data __rte_unused)
3906 {
3907         char *dst = tokens[1], *src = tokens[2];
3908         struct field *fdst, *fsrc;
3909         uint64_t src_val;
3910         uint32_t dst_struct_id, src_struct_id;
3911
3912         CHECK(n_tokens == 3, EINVAL);
3913
3914         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3915         CHECK(fdst, EINVAL);
3916
3917         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3918         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3919         if (fsrc) {
3920                 instr->type = INSTR_ALU_ADD;
3921                 if (dst[0] == 'h' && src[0] != 'h')
3922                         instr->type = INSTR_ALU_ADD_HM;
3923                 if (dst[0] != 'h' && src[0] == 'h')
3924                         instr->type = INSTR_ALU_ADD_MH;
3925                 if (dst[0] == 'h' && src[0] == 'h')
3926                         instr->type = INSTR_ALU_ADD_HH;
3927
3928                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3929                 instr->alu.dst.n_bits = fdst->n_bits;
3930                 instr->alu.dst.offset = fdst->offset / 8;
3931                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3932                 instr->alu.src.n_bits = fsrc->n_bits;
3933                 instr->alu.src.offset = fsrc->offset / 8;
3934                 return 0;
3935         }
3936
3937         /* ADD_MI, ADD_HI. */
3938         src_val = strtoull(src, &src, 0);
3939         CHECK(!src[0], EINVAL);
3940
3941         instr->type = INSTR_ALU_ADD_MI;
3942         if (dst[0] == 'h')
3943                 instr->type = INSTR_ALU_ADD_HI;
3944
3945         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3946         instr->alu.dst.n_bits = fdst->n_bits;
3947         instr->alu.dst.offset = fdst->offset / 8;
3948         instr->alu.src_val = src_val;
3949         return 0;
3950 }
3951
3952 static int
3953 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3954                         struct action *action,
3955                         char **tokens,
3956                         int n_tokens,
3957                         struct instruction *instr,
3958                         struct instruction_data *data __rte_unused)
3959 {
3960         char *dst = tokens[1], *src = tokens[2];
3961         struct field *fdst, *fsrc;
3962         uint64_t src_val;
3963         uint32_t dst_struct_id, src_struct_id;
3964
3965         CHECK(n_tokens == 3, EINVAL);
3966
3967         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3968         CHECK(fdst, EINVAL);
3969
3970         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3971         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3972         if (fsrc) {
3973                 instr->type = INSTR_ALU_SUB;
3974                 if (dst[0] == 'h' && src[0] != 'h')
3975                         instr->type = INSTR_ALU_SUB_HM;
3976                 if (dst[0] != 'h' && src[0] == 'h')
3977                         instr->type = INSTR_ALU_SUB_MH;
3978                 if (dst[0] == 'h' && src[0] == 'h')
3979                         instr->type = INSTR_ALU_SUB_HH;
3980
3981                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3982                 instr->alu.dst.n_bits = fdst->n_bits;
3983                 instr->alu.dst.offset = fdst->offset / 8;
3984                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3985                 instr->alu.src.n_bits = fsrc->n_bits;
3986                 instr->alu.src.offset = fsrc->offset / 8;
3987                 return 0;
3988         }
3989
3990         /* SUB_MI, SUB_HI. */
3991         src_val = strtoull(src, &src, 0);
3992         CHECK(!src[0], EINVAL);
3993
3994         instr->type = INSTR_ALU_SUB_MI;
3995         if (dst[0] == 'h')
3996                 instr->type = INSTR_ALU_SUB_HI;
3997
3998         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3999         instr->alu.dst.n_bits = fdst->n_bits;
4000         instr->alu.dst.offset = fdst->offset / 8;
4001         instr->alu.src_val = src_val;
4002         return 0;
4003 }
4004
4005 static int
4006 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
4007                           struct action *action __rte_unused,
4008                           char **tokens,
4009                           int n_tokens,
4010                           struct instruction *instr,
4011                           struct instruction_data *data __rte_unused)
4012 {
4013         char *dst = tokens[1], *src = tokens[2];
4014         struct header *hdst, *hsrc;
4015         struct field *fdst, *fsrc;
4016
4017         CHECK(n_tokens == 3, EINVAL);
4018
4019         fdst = header_field_parse(p, dst, &hdst);
4020         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4021
4022         /* CKADD_FIELD. */
4023         fsrc = header_field_parse(p, src, &hsrc);
4024         if (fsrc) {
4025                 instr->type = INSTR_ALU_CKADD_FIELD;
4026                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4027                 instr->alu.dst.n_bits = fdst->n_bits;
4028                 instr->alu.dst.offset = fdst->offset / 8;
4029                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4030                 instr->alu.src.n_bits = fsrc->n_bits;
4031                 instr->alu.src.offset = fsrc->offset / 8;
4032                 return 0;
4033         }
4034
4035         /* CKADD_STRUCT, CKADD_STRUCT20. */
4036         hsrc = header_parse(p, src);
4037         CHECK(hsrc, EINVAL);
4038
4039         instr->type = INSTR_ALU_CKADD_STRUCT;
4040         if ((hsrc->st->n_bits / 8) == 20)
4041                 instr->type = INSTR_ALU_CKADD_STRUCT20;
4042
4043         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4044         instr->alu.dst.n_bits = fdst->n_bits;
4045         instr->alu.dst.offset = fdst->offset / 8;
4046         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4047         instr->alu.src.n_bits = hsrc->st->n_bits;
4048         instr->alu.src.offset = 0; /* Unused. */
4049         return 0;
4050 }
4051
4052 static int
4053 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
4054                           struct action *action __rte_unused,
4055                           char **tokens,
4056                           int n_tokens,
4057                           struct instruction *instr,
4058                           struct instruction_data *data __rte_unused)
4059 {
4060         char *dst = tokens[1], *src = tokens[2];
4061         struct header *hdst, *hsrc;
4062         struct field *fdst, *fsrc;
4063
4064         CHECK(n_tokens == 3, EINVAL);
4065
4066         fdst = header_field_parse(p, dst, &hdst);
4067         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4068
4069         fsrc = header_field_parse(p, src, &hsrc);
4070         CHECK(fsrc, EINVAL);
4071
4072         instr->type = INSTR_ALU_CKSUB_FIELD;
4073         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4074         instr->alu.dst.n_bits = fdst->n_bits;
4075         instr->alu.dst.offset = fdst->offset / 8;
4076         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4077         instr->alu.src.n_bits = fsrc->n_bits;
4078         instr->alu.src.offset = fsrc->offset / 8;
4079         return 0;
4080 }
4081
4082 static int
4083 instr_alu_shl_translate(struct rte_swx_pipeline *p,
4084                         struct action *action,
4085                         char **tokens,
4086                         int n_tokens,
4087                         struct instruction *instr,
4088                         struct instruction_data *data __rte_unused)
4089 {
4090         char *dst = tokens[1], *src = tokens[2];
4091         struct field *fdst, *fsrc;
4092         uint64_t src_val;
4093         uint32_t dst_struct_id, src_struct_id;
4094
4095         CHECK(n_tokens == 3, EINVAL);
4096
4097         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4098         CHECK(fdst, EINVAL);
4099
4100         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
4101         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4102         if (fsrc) {
4103                 instr->type = INSTR_ALU_SHL;
4104                 if (dst[0] == 'h' && src[0] != 'h')
4105                         instr->type = INSTR_ALU_SHL_HM;
4106                 if (dst[0] != 'h' && src[0] == 'h')
4107                         instr->type = INSTR_ALU_SHL_MH;
4108                 if (dst[0] == 'h' && src[0] == 'h')
4109                         instr->type = INSTR_ALU_SHL_HH;
4110
4111                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4112                 instr->alu.dst.n_bits = fdst->n_bits;
4113                 instr->alu.dst.offset = fdst->offset / 8;
4114                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4115                 instr->alu.src.n_bits = fsrc->n_bits;
4116                 instr->alu.src.offset = fsrc->offset / 8;
4117                 return 0;
4118         }
4119
4120         /* SHL_MI, SHL_HI. */
4121         src_val = strtoull(src, &src, 0);
4122         CHECK(!src[0], EINVAL);
4123
4124         instr->type = INSTR_ALU_SHL_MI;
4125         if (dst[0] == 'h')
4126                 instr->type = INSTR_ALU_SHL_HI;
4127
4128         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4129         instr->alu.dst.n_bits = fdst->n_bits;
4130         instr->alu.dst.offset = fdst->offset / 8;
4131         instr->alu.src_val = src_val;
4132         return 0;
4133 }
4134
4135 static int
4136 instr_alu_shr_translate(struct rte_swx_pipeline *p,
4137                         struct action *action,
4138                         char **tokens,
4139                         int n_tokens,
4140                         struct instruction *instr,
4141                         struct instruction_data *data __rte_unused)
4142 {
4143         char *dst = tokens[1], *src = tokens[2];
4144         struct field *fdst, *fsrc;
4145         uint64_t src_val;
4146         uint32_t dst_struct_id, src_struct_id;
4147
4148         CHECK(n_tokens == 3, EINVAL);
4149
4150         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4151         CHECK(fdst, EINVAL);
4152
4153         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
4154         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4155         if (fsrc) {
4156                 instr->type = INSTR_ALU_SHR;
4157                 if (dst[0] == 'h' && src[0] != 'h')
4158                         instr->type = INSTR_ALU_SHR_HM;
4159                 if (dst[0] != 'h' && src[0] == 'h')
4160                         instr->type = INSTR_ALU_SHR_MH;
4161                 if (dst[0] == 'h' && src[0] == 'h')
4162                         instr->type = INSTR_ALU_SHR_HH;
4163
4164                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4165                 instr->alu.dst.n_bits = fdst->n_bits;
4166                 instr->alu.dst.offset = fdst->offset / 8;
4167                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4168                 instr->alu.src.n_bits = fsrc->n_bits;
4169                 instr->alu.src.offset = fsrc->offset / 8;
4170                 return 0;
4171         }
4172
4173         /* SHR_MI, SHR_HI. */
4174         src_val = strtoull(src, &src, 0);
4175         CHECK(!src[0], EINVAL);
4176
4177         instr->type = INSTR_ALU_SHR_MI;
4178         if (dst[0] == 'h')
4179                 instr->type = INSTR_ALU_SHR_HI;
4180
4181         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4182         instr->alu.dst.n_bits = fdst->n_bits;
4183         instr->alu.dst.offset = fdst->offset / 8;
4184         instr->alu.src_val = src_val;
4185         return 0;
4186 }
4187
4188 static int
4189 instr_alu_and_translate(struct rte_swx_pipeline *p,
4190                         struct action *action,
4191                         char **tokens,
4192                         int n_tokens,
4193                         struct instruction *instr,
4194                         struct instruction_data *data __rte_unused)
4195 {
4196         char *dst = tokens[1], *src = tokens[2];
4197         struct field *fdst, *fsrc;
4198         uint64_t src_val;
4199         uint32_t dst_struct_id, src_struct_id;
4200
4201         CHECK(n_tokens == 3, EINVAL);
4202
4203         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4204         CHECK(fdst, EINVAL);
4205
4206         /* AND, AND_MH, AND_HM, AND_HH. */
4207         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4208         if (fsrc) {
4209                 instr->type = INSTR_ALU_AND;
4210                 if (dst[0] != 'h' && src[0] == 'h')
4211                         instr->type = INSTR_ALU_AND_MH;
4212                 if (dst[0] == 'h' && src[0] != 'h')
4213                         instr->type = INSTR_ALU_AND_HM;
4214                 if (dst[0] == 'h' && src[0] == 'h')
4215                         instr->type = INSTR_ALU_AND_HH;
4216
4217                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4218                 instr->alu.dst.n_bits = fdst->n_bits;
4219                 instr->alu.dst.offset = fdst->offset / 8;
4220                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4221                 instr->alu.src.n_bits = fsrc->n_bits;
4222                 instr->alu.src.offset = fsrc->offset / 8;
4223                 return 0;
4224         }
4225
4226         /* AND_I. */
4227         src_val = strtoull(src, &src, 0);
4228         CHECK(!src[0], EINVAL);
4229
4230         if (dst[0] == 'h')
4231                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4232
4233         instr->type = INSTR_ALU_AND_I;
4234         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4235         instr->alu.dst.n_bits = fdst->n_bits;
4236         instr->alu.dst.offset = fdst->offset / 8;
4237         instr->alu.src_val = src_val;
4238         return 0;
4239 }
4240
4241 static int
4242 instr_alu_or_translate(struct rte_swx_pipeline *p,
4243                        struct action *action,
4244                        char **tokens,
4245                        int n_tokens,
4246                        struct instruction *instr,
4247                        struct instruction_data *data __rte_unused)
4248 {
4249         char *dst = tokens[1], *src = tokens[2];
4250         struct field *fdst, *fsrc;
4251         uint64_t src_val;
4252         uint32_t dst_struct_id, src_struct_id;
4253
4254         CHECK(n_tokens == 3, EINVAL);
4255
4256         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4257         CHECK(fdst, EINVAL);
4258
4259         /* OR, OR_MH, OR_HM, OR_HH. */
4260         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4261         if (fsrc) {
4262                 instr->type = INSTR_ALU_OR;
4263                 if (dst[0] != 'h' && src[0] == 'h')
4264                         instr->type = INSTR_ALU_OR_MH;
4265                 if (dst[0] == 'h' && src[0] != 'h')
4266                         instr->type = INSTR_ALU_OR_HM;
4267                 if (dst[0] == 'h' && src[0] == 'h')
4268                         instr->type = INSTR_ALU_OR_HH;
4269
4270                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4271                 instr->alu.dst.n_bits = fdst->n_bits;
4272                 instr->alu.dst.offset = fdst->offset / 8;
4273                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4274                 instr->alu.src.n_bits = fsrc->n_bits;
4275                 instr->alu.src.offset = fsrc->offset / 8;
4276                 return 0;
4277         }
4278
4279         /* OR_I. */
4280         src_val = strtoull(src, &src, 0);
4281         CHECK(!src[0], EINVAL);
4282
4283         if (dst[0] == 'h')
4284                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4285
4286         instr->type = INSTR_ALU_OR_I;
4287         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4288         instr->alu.dst.n_bits = fdst->n_bits;
4289         instr->alu.dst.offset = fdst->offset / 8;
4290         instr->alu.src_val = src_val;
4291         return 0;
4292 }
4293
4294 static int
4295 instr_alu_xor_translate(struct rte_swx_pipeline *p,
4296                         struct action *action,
4297                         char **tokens,
4298                         int n_tokens,
4299                         struct instruction *instr,
4300                         struct instruction_data *data __rte_unused)
4301 {
4302         char *dst = tokens[1], *src = tokens[2];
4303         struct field *fdst, *fsrc;
4304         uint64_t src_val;
4305         uint32_t dst_struct_id, src_struct_id;
4306
4307         CHECK(n_tokens == 3, EINVAL);
4308
4309         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4310         CHECK(fdst, EINVAL);
4311
4312         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
4313         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4314         if (fsrc) {
4315                 instr->type = INSTR_ALU_XOR;
4316                 if (dst[0] != 'h' && src[0] == 'h')
4317                         instr->type = INSTR_ALU_XOR_MH;
4318                 if (dst[0] == 'h' && src[0] != 'h')
4319                         instr->type = INSTR_ALU_XOR_HM;
4320                 if (dst[0] == 'h' && src[0] == 'h')
4321                         instr->type = INSTR_ALU_XOR_HH;
4322
4323                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4324                 instr->alu.dst.n_bits = fdst->n_bits;
4325                 instr->alu.dst.offset = fdst->offset / 8;
4326                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4327                 instr->alu.src.n_bits = fsrc->n_bits;
4328                 instr->alu.src.offset = fsrc->offset / 8;
4329                 return 0;
4330         }
4331
4332         /* XOR_I. */
4333         src_val = strtoull(src, &src, 0);
4334         CHECK(!src[0], EINVAL);
4335
4336         if (dst[0] == 'h')
4337                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4338
4339         instr->type = INSTR_ALU_XOR_I;
4340         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4341         instr->alu.dst.n_bits = fdst->n_bits;
4342         instr->alu.dst.offset = fdst->offset / 8;
4343         instr->alu.src_val = src_val;
4344         return 0;
4345 }
4346
4347 static inline void
4348 instr_alu_add_exec(struct rte_swx_pipeline *p)
4349 {
4350         struct thread *t = &p->threads[p->thread_id];
4351         struct instruction *ip = t->ip;
4352
4353         TRACE("[Thread %2u] add\n", p->thread_id);
4354
4355         /* Structs. */
4356         ALU(t, ip, +);
4357
4358         /* Thread. */
4359         thread_ip_inc(p);
4360 }
4361
4362 static inline void
4363 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
4364 {
4365         struct thread *t = &p->threads[p->thread_id];
4366         struct instruction *ip = t->ip;
4367
4368         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
4369
4370         /* Structs. */
4371         ALU_MH(t, ip, +);
4372
4373         /* Thread. */
4374         thread_ip_inc(p);
4375 }
4376
4377 static inline void
4378 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
4379 {
4380         struct thread *t = &p->threads[p->thread_id];
4381         struct instruction *ip = t->ip;
4382
4383         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
4384
4385         /* Structs. */
4386         ALU_HM(t, ip, +);
4387
4388         /* Thread. */
4389         thread_ip_inc(p);
4390 }
4391
4392 static inline void
4393 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
4394 {
4395         struct thread *t = &p->threads[p->thread_id];
4396         struct instruction *ip = t->ip;
4397
4398         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
4399
4400         /* Structs. */
4401         ALU_HH(t, ip, +);
4402
4403         /* Thread. */
4404         thread_ip_inc(p);
4405 }
4406
4407 static inline void
4408 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
4409 {
4410         struct thread *t = &p->threads[p->thread_id];
4411         struct instruction *ip = t->ip;
4412
4413         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
4414
4415         /* Structs. */
4416         ALU_MI(t, ip, +);
4417
4418         /* Thread. */
4419         thread_ip_inc(p);
4420 }
4421
4422 static inline void
4423 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
4424 {
4425         struct thread *t = &p->threads[p->thread_id];
4426         struct instruction *ip = t->ip;
4427
4428         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
4429
4430         /* Structs. */
4431         ALU_HI(t, ip, +);
4432
4433         /* Thread. */
4434         thread_ip_inc(p);
4435 }
4436
4437 static inline void
4438 instr_alu_sub_exec(struct rte_swx_pipeline *p)
4439 {
4440         struct thread *t = &p->threads[p->thread_id];
4441         struct instruction *ip = t->ip;
4442
4443         TRACE("[Thread %2u] sub\n", p->thread_id);
4444
4445         /* Structs. */
4446         ALU(t, ip, -);
4447
4448         /* Thread. */
4449         thread_ip_inc(p);
4450 }
4451
4452 static inline void
4453 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
4454 {
4455         struct thread *t = &p->threads[p->thread_id];
4456         struct instruction *ip = t->ip;
4457
4458         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
4459
4460         /* Structs. */
4461         ALU_MH(t, ip, -);
4462
4463         /* Thread. */
4464         thread_ip_inc(p);
4465 }
4466
4467 static inline void
4468 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4469 {
4470         struct thread *t = &p->threads[p->thread_id];
4471         struct instruction *ip = t->ip;
4472
4473         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4474
4475         /* Structs. */
4476         ALU_HM(t, ip, -);
4477
4478         /* Thread. */
4479         thread_ip_inc(p);
4480 }
4481
4482 static inline void
4483 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4484 {
4485         struct thread *t = &p->threads[p->thread_id];
4486         struct instruction *ip = t->ip;
4487
4488         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4489
4490         /* Structs. */
4491         ALU_HH(t, ip, -);
4492
4493         /* Thread. */
4494         thread_ip_inc(p);
4495 }
4496
4497 static inline void
4498 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4499 {
4500         struct thread *t = &p->threads[p->thread_id];
4501         struct instruction *ip = t->ip;
4502
4503         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4504
4505         /* Structs. */
4506         ALU_MI(t, ip, -);
4507
4508         /* Thread. */
4509         thread_ip_inc(p);
4510 }
4511
4512 static inline void
4513 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4514 {
4515         struct thread *t = &p->threads[p->thread_id];
4516         struct instruction *ip = t->ip;
4517
4518         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4519
4520         /* Structs. */
4521         ALU_HI(t, ip, -);
4522
4523         /* Thread. */
4524         thread_ip_inc(p);
4525 }
4526
4527 static inline void
4528 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4529 {
4530         struct thread *t = &p->threads[p->thread_id];
4531         struct instruction *ip = t->ip;
4532
4533         TRACE("[Thread %2u] shl\n", p->thread_id);
4534
4535         /* Structs. */
4536         ALU(t, ip, <<);
4537
4538         /* Thread. */
4539         thread_ip_inc(p);
4540 }
4541
4542 static inline void
4543 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4544 {
4545         struct thread *t = &p->threads[p->thread_id];
4546         struct instruction *ip = t->ip;
4547
4548         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4549
4550         /* Structs. */
4551         ALU_MH(t, ip, <<);
4552
4553         /* Thread. */
4554         thread_ip_inc(p);
4555 }
4556
4557 static inline void
4558 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4559 {
4560         struct thread *t = &p->threads[p->thread_id];
4561         struct instruction *ip = t->ip;
4562
4563         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4564
4565         /* Structs. */
4566         ALU_HM(t, ip, <<);
4567
4568         /* Thread. */
4569         thread_ip_inc(p);
4570 }
4571
4572 static inline void
4573 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4574 {
4575         struct thread *t = &p->threads[p->thread_id];
4576         struct instruction *ip = t->ip;
4577
4578         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4579
4580         /* Structs. */
4581         ALU_HH(t, ip, <<);
4582
4583         /* Thread. */
4584         thread_ip_inc(p);
4585 }
4586
4587 static inline void
4588 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4589 {
4590         struct thread *t = &p->threads[p->thread_id];
4591         struct instruction *ip = t->ip;
4592
4593         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4594
4595         /* Structs. */
4596         ALU_MI(t, ip, <<);
4597
4598         /* Thread. */
4599         thread_ip_inc(p);
4600 }
4601
4602 static inline void
4603 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4604 {
4605         struct thread *t = &p->threads[p->thread_id];
4606         struct instruction *ip = t->ip;
4607
4608         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4609
4610         /* Structs. */
4611         ALU_HI(t, ip, <<);
4612
4613         /* Thread. */
4614         thread_ip_inc(p);
4615 }
4616
4617 static inline void
4618 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4619 {
4620         struct thread *t = &p->threads[p->thread_id];
4621         struct instruction *ip = t->ip;
4622
4623         TRACE("[Thread %2u] shr\n", p->thread_id);
4624
4625         /* Structs. */
4626         ALU(t, ip, >>);
4627
4628         /* Thread. */
4629         thread_ip_inc(p);
4630 }
4631
4632 static inline void
4633 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4634 {
4635         struct thread *t = &p->threads[p->thread_id];
4636         struct instruction *ip = t->ip;
4637
4638         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4639
4640         /* Structs. */
4641         ALU_MH(t, ip, >>);
4642
4643         /* Thread. */
4644         thread_ip_inc(p);
4645 }
4646
4647 static inline void
4648 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4649 {
4650         struct thread *t = &p->threads[p->thread_id];
4651         struct instruction *ip = t->ip;
4652
4653         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4654
4655         /* Structs. */
4656         ALU_HM(t, ip, >>);
4657
4658         /* Thread. */
4659         thread_ip_inc(p);
4660 }
4661
4662 static inline void
4663 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4664 {
4665         struct thread *t = &p->threads[p->thread_id];
4666         struct instruction *ip = t->ip;
4667
4668         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4669
4670         /* Structs. */
4671         ALU_HH(t, ip, >>);
4672
4673         /* Thread. */
4674         thread_ip_inc(p);
4675 }
4676
4677 static inline void
4678 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4679 {
4680         struct thread *t = &p->threads[p->thread_id];
4681         struct instruction *ip = t->ip;
4682
4683         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4684
4685         /* Structs. */
4686         ALU_MI(t, ip, >>);
4687
4688         /* Thread. */
4689         thread_ip_inc(p);
4690 }
4691
4692 static inline void
4693 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4694 {
4695         struct thread *t = &p->threads[p->thread_id];
4696         struct instruction *ip = t->ip;
4697
4698         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4699
4700         /* Structs. */
4701         ALU_HI(t, ip, >>);
4702
4703         /* Thread. */
4704         thread_ip_inc(p);
4705 }
4706
4707 static inline void
4708 instr_alu_and_exec(struct rte_swx_pipeline *p)
4709 {
4710         struct thread *t = &p->threads[p->thread_id];
4711         struct instruction *ip = t->ip;
4712
4713         TRACE("[Thread %2u] and\n", p->thread_id);
4714
4715         /* Structs. */
4716         ALU(t, ip, &);
4717
4718         /* Thread. */
4719         thread_ip_inc(p);
4720 }
4721
4722 static inline void
4723 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
4724 {
4725         struct thread *t = &p->threads[p->thread_id];
4726         struct instruction *ip = t->ip;
4727
4728         TRACE("[Thread %2u] and (mh)\n", p->thread_id);
4729
4730         /* Structs. */
4731         ALU_MH(t, ip, &);
4732
4733         /* Thread. */
4734         thread_ip_inc(p);
4735 }
4736
4737 static inline void
4738 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
4739 {
4740         struct thread *t = &p->threads[p->thread_id];
4741         struct instruction *ip = t->ip;
4742
4743         TRACE("[Thread %2u] and (hm)\n", p->thread_id);
4744
4745         /* Structs. */
4746         ALU_HM_FAST(t, ip, &);
4747
4748         /* Thread. */
4749         thread_ip_inc(p);
4750 }
4751
4752 static inline void
4753 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
4754 {
4755         struct thread *t = &p->threads[p->thread_id];
4756         struct instruction *ip = t->ip;
4757
4758         TRACE("[Thread %2u] and (hh)\n", p->thread_id);
4759
4760         /* Structs. */
4761         ALU_HH_FAST(t, ip, &);
4762
4763         /* Thread. */
4764         thread_ip_inc(p);
4765 }
4766
4767 static inline void
4768 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4769 {
4770         struct thread *t = &p->threads[p->thread_id];
4771         struct instruction *ip = t->ip;
4772
4773         TRACE("[Thread %2u] and (i)\n", p->thread_id);
4774
4775         /* Structs. */
4776         ALU_I(t, ip, &);
4777
4778         /* Thread. */
4779         thread_ip_inc(p);
4780 }
4781
4782 static inline void
4783 instr_alu_or_exec(struct rte_swx_pipeline *p)
4784 {
4785         struct thread *t = &p->threads[p->thread_id];
4786         struct instruction *ip = t->ip;
4787
4788         TRACE("[Thread %2u] or\n", p->thread_id);
4789
4790         /* Structs. */
4791         ALU(t, ip, |);
4792
4793         /* Thread. */
4794         thread_ip_inc(p);
4795 }
4796
4797 static inline void
4798 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
4799 {
4800         struct thread *t = &p->threads[p->thread_id];
4801         struct instruction *ip = t->ip;
4802
4803         TRACE("[Thread %2u] or (mh)\n", p->thread_id);
4804
4805         /* Structs. */
4806         ALU_MH(t, ip, |);
4807
4808         /* Thread. */
4809         thread_ip_inc(p);
4810 }
4811
4812 static inline void
4813 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
4814 {
4815         struct thread *t = &p->threads[p->thread_id];
4816         struct instruction *ip = t->ip;
4817
4818         TRACE("[Thread %2u] or (hm)\n", p->thread_id);
4819
4820         /* Structs. */
4821         ALU_HM_FAST(t, ip, |);
4822
4823         /* Thread. */
4824         thread_ip_inc(p);
4825 }
4826
4827 static inline void
4828 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
4829 {
4830         struct thread *t = &p->threads[p->thread_id];
4831         struct instruction *ip = t->ip;
4832
4833         TRACE("[Thread %2u] or (hh)\n", p->thread_id);
4834
4835         /* Structs. */
4836         ALU_HH_FAST(t, ip, |);
4837
4838         /* Thread. */
4839         thread_ip_inc(p);
4840 }
4841
4842 static inline void
4843 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4844 {
4845         struct thread *t = &p->threads[p->thread_id];
4846         struct instruction *ip = t->ip;
4847
4848         TRACE("[Thread %2u] or (i)\n", p->thread_id);
4849
4850         /* Structs. */
4851         ALU_I(t, ip, |);
4852
4853         /* Thread. */
4854         thread_ip_inc(p);
4855 }
4856
4857 static inline void
4858 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4859 {
4860         struct thread *t = &p->threads[p->thread_id];
4861         struct instruction *ip = t->ip;
4862
4863         TRACE("[Thread %2u] xor\n", p->thread_id);
4864
4865         /* Structs. */
4866         ALU(t, ip, ^);
4867
4868         /* Thread. */
4869         thread_ip_inc(p);
4870 }
4871
4872 static inline void
4873 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
4874 {
4875         struct thread *t = &p->threads[p->thread_id];
4876         struct instruction *ip = t->ip;
4877
4878         TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
4879
4880         /* Structs. */
4881         ALU_MH(t, ip, ^);
4882
4883         /* Thread. */
4884         thread_ip_inc(p);
4885 }
4886
4887 static inline void
4888 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
4889 {
4890         struct thread *t = &p->threads[p->thread_id];
4891         struct instruction *ip = t->ip;
4892
4893         TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
4894
4895         /* Structs. */
4896         ALU_HM_FAST(t, ip, ^);
4897
4898         /* Thread. */
4899         thread_ip_inc(p);
4900 }
4901
4902 static inline void
4903 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
4904 {
4905         struct thread *t = &p->threads[p->thread_id];
4906         struct instruction *ip = t->ip;
4907
4908         TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
4909
4910         /* Structs. */
4911         ALU_HH_FAST(t, ip, ^);
4912
4913         /* Thread. */
4914         thread_ip_inc(p);
4915 }
4916
4917 static inline void
4918 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4919 {
4920         struct thread *t = &p->threads[p->thread_id];
4921         struct instruction *ip = t->ip;
4922
4923         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
4924
4925         /* Structs. */
4926         ALU_I(t, ip, ^);
4927
4928         /* Thread. */
4929         thread_ip_inc(p);
4930 }
4931
4932 static inline void
4933 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4934 {
4935         struct thread *t = &p->threads[p->thread_id];
4936         struct instruction *ip = t->ip;
4937         uint8_t *dst_struct, *src_struct;
4938         uint16_t *dst16_ptr, dst;
4939         uint64_t *src64_ptr, src64, src64_mask, src;
4940         uint64_t r;
4941
4942         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
4943
4944         /* Structs. */
4945         dst_struct = t->structs[ip->alu.dst.struct_id];
4946         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4947         dst = *dst16_ptr;
4948
4949         src_struct = t->structs[ip->alu.src.struct_id];
4950         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4951         src64 = *src64_ptr;
4952         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4953         src = src64 & src64_mask;
4954
4955         r = dst;
4956         r = ~r & 0xFFFF;
4957
4958         /* The first input (r) is a 16-bit number. The second and the third
4959          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
4960          * three numbers (output r) is a 34-bit number.
4961          */
4962         r += (src >> 32) + (src & 0xFFFFFFFF);
4963
4964         /* The first input is a 16-bit number. The second input is an 18-bit
4965          * number. In the worst case scenario, the sum of the two numbers is a
4966          * 19-bit number.
4967          */
4968         r = (r & 0xFFFF) + (r >> 16);
4969
4970         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4971          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
4972          */
4973         r = (r & 0xFFFF) + (r >> 16);
4974
4975         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4976          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4977          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
4978          * therefore the output r is always a 16-bit number.
4979          */
4980         r = (r & 0xFFFF) + (r >> 16);
4981
4982         r = ~r & 0xFFFF;
4983         r = r ? r : 0xFFFF;
4984
4985         *dst16_ptr = (uint16_t)r;
4986
4987         /* Thread. */
4988         thread_ip_inc(p);
4989 }
4990
4991 static inline void
4992 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4993 {
4994         struct thread *t = &p->threads[p->thread_id];
4995         struct instruction *ip = t->ip;
4996         uint8_t *dst_struct, *src_struct;
4997         uint16_t *dst16_ptr, dst;
4998         uint64_t *src64_ptr, src64, src64_mask, src;
4999         uint64_t r;
5000
5001         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
5002
5003         /* Structs. */
5004         dst_struct = t->structs[ip->alu.dst.struct_id];
5005         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5006         dst = *dst16_ptr;
5007
5008         src_struct = t->structs[ip->alu.src.struct_id];
5009         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
5010         src64 = *src64_ptr;
5011         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
5012         src = src64 & src64_mask;
5013
5014         r = dst;
5015         r = ~r & 0xFFFF;
5016
5017         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
5018          * the following sequence of operations in 2's complement arithmetic:
5019          *    a '- b = (a - b) % 0xFFFF.
5020          *
5021          * In order to prevent an underflow for the below subtraction, in which
5022          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
5023          * minuend), we first add a multiple of the 0xFFFF modulus to the
5024          * minuend. The number we add to the minuend needs to be a 34-bit number
5025          * or higher, so for readability reasons we picked the 36-bit multiple.
5026          * We are effectively turning the 16-bit minuend into a 36-bit number:
5027          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
5028          */
5029         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
5030
5031         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
5032          * result (the output r) is a 36-bit number.
5033          */
5034         r -= (src >> 32) + (src & 0xFFFFFFFF);
5035
5036         /* The first input is a 16-bit number. The second input is a 20-bit
5037          * number. Their sum is a 21-bit number.
5038          */
5039         r = (r & 0xFFFF) + (r >> 16);
5040
5041         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5042          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
5043          */
5044         r = (r & 0xFFFF) + (r >> 16);
5045
5046         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5047          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5048          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5049          * generated, therefore the output r is always a 16-bit number.
5050          */
5051         r = (r & 0xFFFF) + (r >> 16);
5052
5053         r = ~r & 0xFFFF;
5054         r = r ? r : 0xFFFF;
5055
5056         *dst16_ptr = (uint16_t)r;
5057
5058         /* Thread. */
5059         thread_ip_inc(p);
5060 }
5061
5062 static inline void
5063 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
5064 {
5065         struct thread *t = &p->threads[p->thread_id];
5066         struct instruction *ip = t->ip;
5067         uint8_t *dst_struct, *src_struct;
5068         uint16_t *dst16_ptr;
5069         uint32_t *src32_ptr;
5070         uint64_t r0, r1;
5071
5072         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
5073
5074         /* Structs. */
5075         dst_struct = t->structs[ip->alu.dst.struct_id];
5076         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5077
5078         src_struct = t->structs[ip->alu.src.struct_id];
5079         src32_ptr = (uint32_t *)&src_struct[0];
5080
5081         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
5082         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
5083         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
5084         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
5085         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
5086
5087         /* The first input is a 16-bit number. The second input is a 19-bit
5088          * number. Their sum is a 20-bit number.
5089          */
5090         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5091
5092         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5093          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
5094          */
5095         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5096
5097         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5098          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5099          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
5100          * generated, therefore the output r is always a 16-bit number.
5101          */
5102         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5103
5104         r0 = ~r0 & 0xFFFF;
5105         r0 = r0 ? r0 : 0xFFFF;
5106
5107         *dst16_ptr = (uint16_t)r0;
5108
5109         /* Thread. */
5110         thread_ip_inc(p);
5111 }
5112
5113 static inline void
5114 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
5115 {
5116         struct thread *t = &p->threads[p->thread_id];
5117         struct instruction *ip = t->ip;
5118         uint8_t *dst_struct, *src_struct;
5119         uint16_t *dst16_ptr;
5120         uint32_t *src32_ptr;
5121         uint64_t r = 0;
5122         uint32_t i;
5123
5124         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
5125
5126         /* Structs. */
5127         dst_struct = t->structs[ip->alu.dst.struct_id];
5128         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5129
5130         src_struct = t->structs[ip->alu.src.struct_id];
5131         src32_ptr = (uint32_t *)&src_struct[0];
5132
5133         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
5134          * Therefore, in the worst case scenario, a 35-bit number is added to a
5135          * 16-bit number (the input r), so the output r is 36-bit number.
5136          */
5137         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
5138                 r += *src32_ptr;
5139
5140         /* The first input is a 16-bit number. The second input is a 20-bit
5141          * number. Their sum is a 21-bit number.
5142          */
5143         r = (r & 0xFFFF) + (r >> 16);
5144
5145         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5146          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
5147          */
5148         r = (r & 0xFFFF) + (r >> 16);
5149
5150         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5151          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5152          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5153          * generated, therefore the output r is always a 16-bit number.
5154          */
5155         r = (r & 0xFFFF) + (r >> 16);
5156
5157         r = ~r & 0xFFFF;
5158         r = r ? r : 0xFFFF;
5159
5160         *dst16_ptr = (uint16_t)r;
5161
5162         /* Thread. */
5163         thread_ip_inc(p);
5164 }
5165
5166 /*
5167  * Register array.
5168  */
5169 static struct regarray *
5170 regarray_find(struct rte_swx_pipeline *p, const char *name);
5171
5172 static int
5173 instr_regprefetch_translate(struct rte_swx_pipeline *p,
5174                       struct action *action,
5175                       char **tokens,
5176                       int n_tokens,
5177                       struct instruction *instr,
5178                       struct instruction_data *data __rte_unused)
5179 {
5180         char *regarray = tokens[1], *idx = tokens[2];
5181         struct regarray *r;
5182         struct field *fidx;
5183         uint32_t idx_struct_id, idx_val;
5184
5185         CHECK(n_tokens == 3, EINVAL);
5186
5187         r = regarray_find(p, regarray);
5188         CHECK(r, EINVAL);
5189
5190         /* REGPREFETCH_RH, REGPREFETCH_RM. */
5191         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5192         if (fidx) {
5193                 instr->type = INSTR_REGPREFETCH_RM;
5194                 if (idx[0] == 'h')
5195                         instr->type = INSTR_REGPREFETCH_RH;
5196
5197                 instr->regarray.regarray_id = r->id;
5198                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5199                 instr->regarray.idx.n_bits = fidx->n_bits;
5200                 instr->regarray.idx.offset = fidx->offset / 8;
5201                 instr->regarray.dstsrc_val = 0; /* Unused. */
5202                 return 0;
5203         }
5204
5205         /* REGPREFETCH_RI. */
5206         idx_val = strtoul(idx, &idx, 0);
5207         CHECK(!idx[0], EINVAL);
5208
5209         instr->type = INSTR_REGPREFETCH_RI;
5210         instr->regarray.regarray_id = r->id;
5211         instr->regarray.idx_val = idx_val;
5212         instr->regarray.dstsrc_val = 0; /* Unused. */
5213         return 0;
5214 }
5215
5216 static int
5217 instr_regrd_translate(struct rte_swx_pipeline *p,
5218                       struct action *action,
5219                       char **tokens,
5220                       int n_tokens,
5221                       struct instruction *instr,
5222                       struct instruction_data *data __rte_unused)
5223 {
5224         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
5225         struct regarray *r;
5226         struct field *fdst, *fidx;
5227         uint32_t dst_struct_id, idx_struct_id, idx_val;
5228
5229         CHECK(n_tokens == 4, EINVAL);
5230
5231         r = regarray_find(p, regarray);
5232         CHECK(r, EINVAL);
5233
5234         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
5235         CHECK(fdst, EINVAL);
5236
5237         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
5238         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5239         if (fidx) {
5240                 instr->type = INSTR_REGRD_MRM;
5241                 if (dst[0] == 'h' && idx[0] != 'h')
5242                         instr->type = INSTR_REGRD_HRM;
5243                 if (dst[0] != 'h' && idx[0] == 'h')
5244                         instr->type = INSTR_REGRD_MRH;
5245                 if (dst[0] == 'h' && idx[0] == 'h')
5246                         instr->type = INSTR_REGRD_HRH;
5247
5248                 instr->regarray.regarray_id = r->id;
5249                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5250                 instr->regarray.idx.n_bits = fidx->n_bits;
5251                 instr->regarray.idx.offset = fidx->offset / 8;
5252                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5253                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
5254                 instr->regarray.dstsrc.offset = fdst->offset / 8;
5255                 return 0;
5256         }
5257
5258         /* REGRD_MRI, REGRD_HRI. */
5259         idx_val = strtoul(idx, &idx, 0);
5260         CHECK(!idx[0], EINVAL);
5261
5262         instr->type = INSTR_REGRD_MRI;
5263         if (dst[0] == 'h')
5264                 instr->type = INSTR_REGRD_HRI;
5265
5266         instr->regarray.regarray_id = r->id;
5267         instr->regarray.idx_val = idx_val;
5268         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5269         instr->regarray.dstsrc.n_bits = fdst->n_bits;
5270         instr->regarray.dstsrc.offset = fdst->offset / 8;
5271         return 0;
5272 }
5273
5274 static int
5275 instr_regwr_translate(struct rte_swx_pipeline *p,
5276                       struct action *action,
5277                       char **tokens,
5278                       int n_tokens,
5279                       struct instruction *instr,
5280                       struct instruction_data *data __rte_unused)
5281 {
5282         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5283         struct regarray *r;
5284         struct field *fidx, *fsrc;
5285         uint64_t src_val;
5286         uint32_t idx_struct_id, idx_val, src_struct_id;
5287
5288         CHECK(n_tokens == 4, EINVAL);
5289
5290         r = regarray_find(p, regarray);
5291         CHECK(r, EINVAL);
5292
5293         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
5294         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5295         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5296         if (fidx && fsrc) {
5297                 instr->type = INSTR_REGWR_RMM;
5298                 if (idx[0] == 'h' && src[0] != 'h')
5299                         instr->type = INSTR_REGWR_RHM;
5300                 if (idx[0] != 'h' && src[0] == 'h')
5301                         instr->type = INSTR_REGWR_RMH;
5302                 if (idx[0] == 'h' && src[0] == 'h')
5303                         instr->type = INSTR_REGWR_RHH;
5304
5305                 instr->regarray.regarray_id = r->id;
5306                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5307                 instr->regarray.idx.n_bits = fidx->n_bits;
5308                 instr->regarray.idx.offset = fidx->offset / 8;
5309                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5310                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5311                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5312                 return 0;
5313         }
5314
5315         /* REGWR_RHI, REGWR_RMI. */
5316         if (fidx && !fsrc) {
5317                 src_val = strtoull(src, &src, 0);
5318                 CHECK(!src[0], EINVAL);
5319
5320                 instr->type = INSTR_REGWR_RMI;
5321                 if (idx[0] == 'h')
5322                         instr->type = INSTR_REGWR_RHI;
5323
5324                 instr->regarray.regarray_id = r->id;
5325                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5326                 instr->regarray.idx.n_bits = fidx->n_bits;
5327                 instr->regarray.idx.offset = fidx->offset / 8;
5328                 instr->regarray.dstsrc_val = src_val;
5329                 return 0;
5330         }
5331
5332         /* REGWR_RIH, REGWR_RIM. */
5333         if (!fidx && fsrc) {
5334                 idx_val = strtoul(idx, &idx, 0);
5335                 CHECK(!idx[0], EINVAL);
5336
5337                 instr->type = INSTR_REGWR_RIM;
5338                 if (src[0] == 'h')
5339                         instr->type = INSTR_REGWR_RIH;
5340
5341                 instr->regarray.regarray_id = r->id;
5342                 instr->regarray.idx_val = idx_val;
5343                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5344                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5345                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5346                 return 0;
5347         }
5348
5349         /* REGWR_RII. */
5350         src_val = strtoull(src, &src, 0);
5351         CHECK(!src[0], EINVAL);
5352
5353         idx_val = strtoul(idx, &idx, 0);
5354         CHECK(!idx[0], EINVAL);
5355
5356         instr->type = INSTR_REGWR_RII;
5357         instr->regarray.idx_val = idx_val;
5358         instr->regarray.dstsrc_val = src_val;
5359
5360         return 0;
5361 }
5362
5363 static int
5364 instr_regadd_translate(struct rte_swx_pipeline *p,
5365                        struct action *action,
5366                        char **tokens,
5367                        int n_tokens,
5368                        struct instruction *instr,
5369                        struct instruction_data *data __rte_unused)
5370 {
5371         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5372         struct regarray *r;
5373         struct field *fidx, *fsrc;
5374         uint64_t src_val;
5375         uint32_t idx_struct_id, idx_val, src_struct_id;
5376
5377         CHECK(n_tokens == 4, EINVAL);
5378
5379         r = regarray_find(p, regarray);
5380         CHECK(r, EINVAL);
5381
5382         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
5383         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5384         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5385         if (fidx && fsrc) {
5386                 instr->type = INSTR_REGADD_RMM;
5387                 if (idx[0] == 'h' && src[0] != 'h')
5388                         instr->type = INSTR_REGADD_RHM;
5389                 if (idx[0] != 'h' && src[0] == 'h')
5390                         instr->type = INSTR_REGADD_RMH;
5391                 if (idx[0] == 'h' && src[0] == 'h')
5392                         instr->type = INSTR_REGADD_RHH;
5393
5394                 instr->regarray.regarray_id = r->id;
5395                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5396                 instr->regarray.idx.n_bits = fidx->n_bits;
5397                 instr->regarray.idx.offset = fidx->offset / 8;
5398                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5399                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5400                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5401                 return 0;
5402         }
5403
5404         /* REGADD_RHI, REGADD_RMI. */
5405         if (fidx && !fsrc) {
5406                 src_val = strtoull(src, &src, 0);
5407                 CHECK(!src[0], EINVAL);
5408
5409                 instr->type = INSTR_REGADD_RMI;
5410                 if (idx[0] == 'h')
5411                         instr->type = INSTR_REGADD_RHI;
5412
5413                 instr->regarray.regarray_id = r->id;
5414                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5415                 instr->regarray.idx.n_bits = fidx->n_bits;
5416                 instr->regarray.idx.offset = fidx->offset / 8;
5417                 instr->regarray.dstsrc_val = src_val;
5418                 return 0;
5419         }
5420
5421         /* REGADD_RIH, REGADD_RIM. */
5422         if (!fidx && fsrc) {
5423                 idx_val = strtoul(idx, &idx, 0);
5424                 CHECK(!idx[0], EINVAL);
5425
5426                 instr->type = INSTR_REGADD_RIM;
5427                 if (src[0] == 'h')
5428                         instr->type = INSTR_REGADD_RIH;
5429
5430                 instr->regarray.regarray_id = r->id;
5431                 instr->regarray.idx_val = idx_val;
5432                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5433                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5434                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5435                 return 0;
5436         }
5437
5438         /* REGADD_RII. */
5439         src_val = strtoull(src, &src, 0);
5440         CHECK(!src[0], EINVAL);
5441
5442         idx_val = strtoul(idx, &idx, 0);
5443         CHECK(!idx[0], EINVAL);
5444
5445         instr->type = INSTR_REGADD_RII;
5446         instr->regarray.idx_val = idx_val;
5447         instr->regarray.dstsrc_val = src_val;
5448         return 0;
5449 }
5450
5451 static inline uint64_t *
5452 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
5453 {
5454         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5455         return r->regarray;
5456 }
5457
5458 static inline uint64_t
5459 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5460 {
5461         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5462
5463         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5464         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5465         uint64_t idx64 = *idx64_ptr;
5466         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
5467         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5468
5469         return idx;
5470 }
5471
5472 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5473
5474 static inline uint64_t
5475 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5476 {
5477         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5478
5479         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5480         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5481         uint64_t idx64 = *idx64_ptr;
5482         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
5483
5484         return idx;
5485 }
5486
5487 #else
5488
5489 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
5490
5491 #endif
5492
5493 static inline uint64_t
5494 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5495 {
5496         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5497
5498         uint64_t idx = ip->regarray.idx_val & r->size_mask;
5499
5500         return idx;
5501 }
5502
5503 static inline uint64_t
5504 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
5505 {
5506         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5507         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5508         uint64_t src64 = *src64_ptr;
5509         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5510         uint64_t src = src64 & src64_mask;
5511
5512         return src;
5513 }
5514
5515 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5516
5517 static inline uint64_t
5518 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
5519 {
5520         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5521         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5522         uint64_t src64 = *src64_ptr;
5523         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
5524
5525         return src;
5526 }
5527
5528 #else
5529
5530 #define instr_regarray_src_nbo instr_regarray_src_hbo
5531
5532 #endif
5533
5534 static inline void
5535 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5536 {
5537         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5538         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5539         uint64_t dst64 = *dst64_ptr;
5540         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5541
5542         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5543
5544 }
5545
5546 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5547
5548 static inline void
5549 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5550 {
5551         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5552         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5553         uint64_t dst64 = *dst64_ptr;
5554         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5555
5556         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
5557         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5558 }
5559
5560 #else
5561
5562 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
5563
5564 #endif
5565
5566 static inline void
5567 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
5568 {
5569         struct thread *t = &p->threads[p->thread_id];
5570         struct instruction *ip = t->ip;
5571         uint64_t *regarray, idx;
5572
5573         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
5574
5575         /* Structs. */
5576         regarray = instr_regarray_regarray(p, ip);
5577         idx = instr_regarray_idx_nbo(p, t, ip);
5578         rte_prefetch0(&regarray[idx]);
5579
5580         /* Thread. */
5581         thread_ip_inc(p);
5582 }
5583
5584 static inline void
5585 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
5586 {
5587         struct thread *t = &p->threads[p->thread_id];
5588         struct instruction *ip = t->ip;
5589         uint64_t *regarray, idx;
5590
5591         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
5592
5593         /* Structs. */
5594         regarray = instr_regarray_regarray(p, ip);
5595         idx = instr_regarray_idx_hbo(p, t, ip);
5596         rte_prefetch0(&regarray[idx]);
5597
5598         /* Thread. */
5599         thread_ip_inc(p);
5600 }
5601
5602 static inline void
5603 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
5604 {
5605         struct thread *t = &p->threads[p->thread_id];
5606         struct instruction *ip = t->ip;
5607         uint64_t *regarray, idx;
5608
5609         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
5610
5611         /* Structs. */
5612         regarray = instr_regarray_regarray(p, ip);
5613         idx = instr_regarray_idx_imm(p, ip);
5614         rte_prefetch0(&regarray[idx]);
5615
5616         /* Thread. */
5617         thread_ip_inc(p);
5618 }
5619
5620 static inline void
5621 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
5622 {
5623         struct thread *t = &p->threads[p->thread_id];
5624         struct instruction *ip = t->ip;
5625         uint64_t *regarray, idx;
5626
5627         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
5628
5629         /* Structs. */
5630         regarray = instr_regarray_regarray(p, ip);
5631         idx = instr_regarray_idx_nbo(p, t, ip);
5632         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5633
5634         /* Thread. */
5635         thread_ip_inc(p);
5636 }
5637
5638 static inline void
5639 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
5640 {
5641         struct thread *t = &p->threads[p->thread_id];
5642         struct instruction *ip = t->ip;
5643         uint64_t *regarray, idx;
5644
5645         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
5646
5647         /* Structs. */
5648         regarray = instr_regarray_regarray(p, ip);
5649         idx = instr_regarray_idx_hbo(p, t, ip);
5650         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5651
5652         /* Thread. */
5653         thread_ip_inc(p);
5654 }
5655
5656 static inline void
5657 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
5658 {
5659         struct thread *t = &p->threads[p->thread_id];
5660         struct instruction *ip = t->ip;
5661         uint64_t *regarray, idx;
5662
5663         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
5664
5665         /* Structs. */
5666         regarray = instr_regarray_regarray(p, ip);
5667         idx = instr_regarray_idx_nbo(p, t, ip);
5668         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5669
5670         /* Thread. */
5671         thread_ip_inc(p);
5672 }
5673
5674 static inline void
5675 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
5676 {
5677         struct thread *t = &p->threads[p->thread_id];
5678         struct instruction *ip = t->ip;
5679         uint64_t *regarray, idx;
5680
5681         /* Structs. */
5682         regarray = instr_regarray_regarray(p, ip);
5683         idx = instr_regarray_idx_hbo(p, t, ip);
5684         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5685
5686         /* Thread. */
5687         thread_ip_inc(p);
5688 }
5689
5690 static inline void
5691 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
5692 {
5693         struct thread *t = &p->threads[p->thread_id];
5694         struct instruction *ip = t->ip;
5695         uint64_t *regarray, idx;
5696
5697         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
5698
5699         /* Structs. */
5700         regarray = instr_regarray_regarray(p, ip);
5701         idx = instr_regarray_idx_imm(p, ip);
5702         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5703
5704         /* Thread. */
5705         thread_ip_inc(p);
5706 }
5707
5708 static inline void
5709 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
5710 {
5711         struct thread *t = &p->threads[p->thread_id];
5712         struct instruction *ip = t->ip;
5713         uint64_t *regarray, idx;
5714
5715         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
5716
5717         /* Structs. */
5718         regarray = instr_regarray_regarray(p, ip);
5719         idx = instr_regarray_idx_imm(p, ip);
5720         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5721
5722         /* Thread. */
5723         thread_ip_inc(p);
5724 }
5725
5726 static inline void
5727 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
5728 {
5729         struct thread *t = &p->threads[p->thread_id];
5730         struct instruction *ip = t->ip;
5731         uint64_t *regarray, idx, src;
5732
5733         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
5734
5735         /* Structs. */
5736         regarray = instr_regarray_regarray(p, ip);
5737         idx = instr_regarray_idx_nbo(p, t, ip);
5738         src = instr_regarray_src_nbo(t, ip);
5739         regarray[idx] = src;
5740
5741         /* Thread. */
5742         thread_ip_inc(p);
5743 }
5744
5745 static inline void
5746 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
5747 {
5748         struct thread *t = &p->threads[p->thread_id];
5749         struct instruction *ip = t->ip;
5750         uint64_t *regarray, idx, src;
5751
5752         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
5753
5754         /* Structs. */
5755         regarray = instr_regarray_regarray(p, ip);
5756         idx = instr_regarray_idx_nbo(p, t, ip);
5757         src = instr_regarray_src_hbo(t, ip);
5758         regarray[idx] = src;
5759
5760         /* Thread. */
5761         thread_ip_inc(p);
5762 }
5763
5764 static inline void
5765 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
5766 {
5767         struct thread *t = &p->threads[p->thread_id];
5768         struct instruction *ip = t->ip;
5769         uint64_t *regarray, idx, src;
5770
5771         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
5772
5773         /* Structs. */
5774         regarray = instr_regarray_regarray(p, ip);
5775         idx = instr_regarray_idx_hbo(p, t, ip);
5776         src = instr_regarray_src_nbo(t, ip);
5777         regarray[idx] = src;
5778
5779         /* Thread. */
5780         thread_ip_inc(p);
5781 }
5782
5783 static inline void
5784 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
5785 {
5786         struct thread *t = &p->threads[p->thread_id];
5787         struct instruction *ip = t->ip;
5788         uint64_t *regarray, idx, src;
5789
5790         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
5791
5792         /* Structs. */
5793         regarray = instr_regarray_regarray(p, ip);
5794         idx = instr_regarray_idx_hbo(p, t, ip);
5795         src = instr_regarray_src_hbo(t, ip);
5796         regarray[idx] = src;
5797
5798         /* Thread. */
5799         thread_ip_inc(p);
5800 }
5801
5802 static inline void
5803 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
5804 {
5805         struct thread *t = &p->threads[p->thread_id];
5806         struct instruction *ip = t->ip;
5807         uint64_t *regarray, idx, src;
5808
5809         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
5810
5811         /* Structs. */
5812         regarray = instr_regarray_regarray(p, ip);
5813         idx = instr_regarray_idx_nbo(p, t, ip);
5814         src = ip->regarray.dstsrc_val;
5815         regarray[idx] = src;
5816
5817         /* Thread. */
5818         thread_ip_inc(p);
5819 }
5820
5821 static inline void
5822 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
5823 {
5824         struct thread *t = &p->threads[p->thread_id];
5825         struct instruction *ip = t->ip;
5826         uint64_t *regarray, idx, src;
5827
5828         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
5829
5830         /* Structs. */
5831         regarray = instr_regarray_regarray(p, ip);
5832         idx = instr_regarray_idx_hbo(p, t, ip);
5833         src = ip->regarray.dstsrc_val;
5834         regarray[idx] = src;
5835
5836         /* Thread. */
5837         thread_ip_inc(p);
5838 }
5839
5840 static inline void
5841 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
5842 {
5843         struct thread *t = &p->threads[p->thread_id];
5844         struct instruction *ip = t->ip;
5845         uint64_t *regarray, idx, src;
5846
5847         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
5848
5849         /* Structs. */
5850         regarray = instr_regarray_regarray(p, ip);
5851         idx = instr_regarray_idx_imm(p, ip);
5852         src = instr_regarray_src_nbo(t, ip);
5853         regarray[idx] = src;
5854
5855         /* Thread. */
5856         thread_ip_inc(p);
5857 }
5858
5859 static inline void
5860 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
5861 {
5862         struct thread *t = &p->threads[p->thread_id];
5863         struct instruction *ip = t->ip;
5864         uint64_t *regarray, idx, src;
5865
5866         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
5867
5868         /* Structs. */
5869         regarray = instr_regarray_regarray(p, ip);
5870         idx = instr_regarray_idx_imm(p, ip);
5871         src = instr_regarray_src_hbo(t, ip);
5872         regarray[idx] = src;
5873
5874         /* Thread. */
5875         thread_ip_inc(p);
5876 }
5877
5878 static inline void
5879 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
5880 {
5881         struct thread *t = &p->threads[p->thread_id];
5882         struct instruction *ip = t->ip;
5883         uint64_t *regarray, idx, src;
5884
5885         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
5886
5887         /* Structs. */
5888         regarray = instr_regarray_regarray(p, ip);
5889         idx = instr_regarray_idx_imm(p, ip);
5890         src = ip->regarray.dstsrc_val;
5891         regarray[idx] = src;
5892
5893         /* Thread. */
5894         thread_ip_inc(p);
5895 }
5896
5897 static inline void
5898 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
5899 {
5900         struct thread *t = &p->threads[p->thread_id];
5901         struct instruction *ip = t->ip;
5902         uint64_t *regarray, idx, src;
5903
5904         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
5905
5906         /* Structs. */
5907         regarray = instr_regarray_regarray(p, ip);
5908         idx = instr_regarray_idx_nbo(p, t, ip);
5909         src = instr_regarray_src_nbo(t, ip);
5910         regarray[idx] += src;
5911
5912         /* Thread. */
5913         thread_ip_inc(p);
5914 }
5915
5916 static inline void
5917 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
5918 {
5919         struct thread *t = &p->threads[p->thread_id];
5920         struct instruction *ip = t->ip;
5921         uint64_t *regarray, idx, src;
5922
5923         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
5924
5925         /* Structs. */
5926         regarray = instr_regarray_regarray(p, ip);
5927         idx = instr_regarray_idx_nbo(p, t, ip);
5928         src = instr_regarray_src_hbo(t, ip);
5929         regarray[idx] += src;
5930
5931         /* Thread. */
5932         thread_ip_inc(p);
5933 }
5934
5935 static inline void
5936 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
5937 {
5938         struct thread *t = &p->threads[p->thread_id];
5939         struct instruction *ip = t->ip;
5940         uint64_t *regarray, idx, src;
5941
5942         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
5943
5944         /* Structs. */
5945         regarray = instr_regarray_regarray(p, ip);
5946         idx = instr_regarray_idx_hbo(p, t, ip);
5947         src = instr_regarray_src_nbo(t, ip);
5948         regarray[idx] += src;
5949
5950         /* Thread. */
5951         thread_ip_inc(p);
5952 }
5953
5954 static inline void
5955 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
5956 {
5957         struct thread *t = &p->threads[p->thread_id];
5958         struct instruction *ip = t->ip;
5959         uint64_t *regarray, idx, src;
5960
5961         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
5962
5963         /* Structs. */
5964         regarray = instr_regarray_regarray(p, ip);
5965         idx = instr_regarray_idx_hbo(p, t, ip);
5966         src = instr_regarray_src_hbo(t, ip);
5967         regarray[idx] += src;
5968
5969         /* Thread. */
5970         thread_ip_inc(p);
5971 }
5972
5973 static inline void
5974 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
5975 {
5976         struct thread *t = &p->threads[p->thread_id];
5977         struct instruction *ip = t->ip;
5978         uint64_t *regarray, idx, src;
5979
5980         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
5981
5982         /* Structs. */
5983         regarray = instr_regarray_regarray(p, ip);
5984         idx = instr_regarray_idx_nbo(p, t, ip);
5985         src = ip->regarray.dstsrc_val;
5986         regarray[idx] += src;
5987
5988         /* Thread. */
5989         thread_ip_inc(p);
5990 }
5991
5992 static inline void
5993 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
5994 {
5995         struct thread *t = &p->threads[p->thread_id];
5996         struct instruction *ip = t->ip;
5997         uint64_t *regarray, idx, src;
5998
5999         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
6000
6001         /* Structs. */
6002         regarray = instr_regarray_regarray(p, ip);
6003         idx = instr_regarray_idx_hbo(p, t, ip);
6004         src = ip->regarray.dstsrc_val;
6005         regarray[idx] += src;
6006
6007         /* Thread. */
6008         thread_ip_inc(p);
6009 }
6010
6011 static inline void
6012 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
6013 {
6014         struct thread *t = &p->threads[p->thread_id];
6015         struct instruction *ip = t->ip;
6016         uint64_t *regarray, idx, src;
6017
6018         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
6019
6020         /* Structs. */
6021         regarray = instr_regarray_regarray(p, ip);
6022         idx = instr_regarray_idx_imm(p, ip);
6023         src = instr_regarray_src_nbo(t, ip);
6024         regarray[idx] += src;
6025
6026         /* Thread. */
6027         thread_ip_inc(p);
6028 }
6029
6030 static inline void
6031 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
6032 {
6033         struct thread *t = &p->threads[p->thread_id];
6034         struct instruction *ip = t->ip;
6035         uint64_t *regarray, idx, src;
6036
6037         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
6038
6039         /* Structs. */
6040         regarray = instr_regarray_regarray(p, ip);
6041         idx = instr_regarray_idx_imm(p, ip);
6042         src = instr_regarray_src_hbo(t, ip);
6043         regarray[idx] += src;
6044
6045         /* Thread. */
6046         thread_ip_inc(p);
6047 }
6048
6049 static inline void
6050 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
6051 {
6052         struct thread *t = &p->threads[p->thread_id];
6053         struct instruction *ip = t->ip;
6054         uint64_t *regarray, idx, src;
6055
6056         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
6057
6058         /* Structs. */
6059         regarray = instr_regarray_regarray(p, ip);
6060         idx = instr_regarray_idx_imm(p, ip);
6061         src = ip->regarray.dstsrc_val;
6062         regarray[idx] += src;
6063
6064         /* Thread. */
6065         thread_ip_inc(p);
6066 }
6067
6068 /*
6069  * metarray.
6070  */
6071 static struct metarray *
6072 metarray_find(struct rte_swx_pipeline *p, const char *name);
6073
6074 static int
6075 instr_metprefetch_translate(struct rte_swx_pipeline *p,
6076                             struct action *action,
6077                             char **tokens,
6078                             int n_tokens,
6079                             struct instruction *instr,
6080                             struct instruction_data *data __rte_unused)
6081 {
6082         char *metarray = tokens[1], *idx = tokens[2];
6083         struct metarray *m;
6084         struct field *fidx;
6085         uint32_t idx_struct_id, idx_val;
6086
6087         CHECK(n_tokens == 3, EINVAL);
6088
6089         m = metarray_find(p, metarray);
6090         CHECK(m, EINVAL);
6091
6092         /* METPREFETCH_H, METPREFETCH_M. */
6093         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6094         if (fidx) {
6095                 instr->type = INSTR_METPREFETCH_M;
6096                 if (idx[0] == 'h')
6097                         instr->type = INSTR_METPREFETCH_H;
6098
6099                 instr->meter.metarray_id = m->id;
6100                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6101                 instr->meter.idx.n_bits = fidx->n_bits;
6102                 instr->meter.idx.offset = fidx->offset / 8;
6103                 return 0;
6104         }
6105
6106         /* METPREFETCH_I. */
6107         idx_val = strtoul(idx, &idx, 0);
6108         CHECK(!idx[0], EINVAL);
6109
6110         instr->type = INSTR_METPREFETCH_I;
6111         instr->meter.metarray_id = m->id;
6112         instr->meter.idx_val = idx_val;
6113         return 0;
6114 }
6115
6116 static int
6117 instr_meter_translate(struct rte_swx_pipeline *p,
6118                       struct action *action,
6119                       char **tokens,
6120                       int n_tokens,
6121                       struct instruction *instr,
6122                       struct instruction_data *data __rte_unused)
6123 {
6124         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
6125         char *color_in = tokens[4], *color_out = tokens[5];
6126         struct metarray *m;
6127         struct field *fidx, *flength, *fcin, *fcout;
6128         uint32_t idx_struct_id, length_struct_id;
6129         uint32_t color_in_struct_id, color_out_struct_id;
6130
6131         CHECK(n_tokens == 6, EINVAL);
6132
6133         m = metarray_find(p, metarray);
6134         CHECK(m, EINVAL);
6135
6136         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6137
6138         flength = struct_field_parse(p, action, length, &length_struct_id);
6139         CHECK(flength, EINVAL);
6140
6141         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
6142
6143         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
6144         CHECK(fcout, EINVAL);
6145
6146         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
6147         if (fidx && fcin) {
6148                 instr->type = INSTR_METER_MMM;
6149                 if (idx[0] == 'h' && length[0] == 'h')
6150                         instr->type = INSTR_METER_HHM;
6151                 if (idx[0] == 'h' && length[0] != 'h')
6152                         instr->type = INSTR_METER_HMM;
6153                 if (idx[0] != 'h' && length[0] == 'h')
6154                         instr->type = INSTR_METER_MHM;
6155
6156                 instr->meter.metarray_id = m->id;
6157
6158                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6159                 instr->meter.idx.n_bits = fidx->n_bits;
6160                 instr->meter.idx.offset = fidx->offset / 8;
6161
6162                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6163                 instr->meter.length.n_bits = flength->n_bits;
6164                 instr->meter.length.offset = flength->offset / 8;
6165
6166                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6167                 instr->meter.color_in.n_bits = fcin->n_bits;
6168                 instr->meter.color_in.offset = fcin->offset / 8;
6169
6170                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6171                 instr->meter.color_out.n_bits = fcout->n_bits;
6172                 instr->meter.color_out.offset = fcout->offset / 8;
6173
6174                 return 0;
6175         }
6176
6177         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
6178         if (fidx && !fcin) {
6179                 uint32_t color_in_val = strtoul(color_in, &color_in, 0);
6180                 CHECK(!color_in[0], EINVAL);
6181
6182                 instr->type = INSTR_METER_MMI;
6183                 if (idx[0] == 'h' && length[0] == 'h')
6184                         instr->type = INSTR_METER_HHI;
6185                 if (idx[0] == 'h' && length[0] != 'h')
6186                         instr->type = INSTR_METER_HMI;
6187                 if (idx[0] != 'h' && length[0] == 'h')
6188                         instr->type = INSTR_METER_MHI;
6189
6190                 instr->meter.metarray_id = m->id;
6191
6192                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6193                 instr->meter.idx.n_bits = fidx->n_bits;
6194                 instr->meter.idx.offset = fidx->offset / 8;
6195
6196                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6197                 instr->meter.length.n_bits = flength->n_bits;
6198                 instr->meter.length.offset = flength->offset / 8;
6199
6200                 instr->meter.color_in_val = color_in_val;
6201
6202                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6203                 instr->meter.color_out.n_bits = fcout->n_bits;
6204                 instr->meter.color_out.offset = fcout->offset / 8;
6205
6206                 return 0;
6207         }
6208
6209         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
6210         if (!fidx && fcin) {
6211                 uint32_t idx_val;
6212
6213                 idx_val = strtoul(idx, &idx, 0);
6214                 CHECK(!idx[0], EINVAL);
6215
6216                 instr->type = INSTR_METER_IMM;
6217                 if (length[0] == 'h')
6218                         instr->type = INSTR_METER_IHM;
6219
6220                 instr->meter.metarray_id = m->id;
6221
6222                 instr->meter.idx_val = idx_val;
6223
6224                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6225                 instr->meter.length.n_bits = flength->n_bits;
6226                 instr->meter.length.offset = flength->offset / 8;
6227
6228                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6229                 instr->meter.color_in.n_bits = fcin->n_bits;
6230                 instr->meter.color_in.offset = fcin->offset / 8;
6231
6232                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6233                 instr->meter.color_out.n_bits = fcout->n_bits;
6234                 instr->meter.color_out.offset = fcout->offset / 8;
6235
6236                 return 0;
6237         }
6238
6239         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
6240         if (!fidx && !fcin) {
6241                 uint32_t idx_val, color_in_val;
6242
6243                 idx_val = strtoul(idx, &idx, 0);
6244                 CHECK(!idx[0], EINVAL);
6245
6246                 color_in_val = strtoul(color_in, &color_in, 0);
6247                 CHECK(!color_in[0], EINVAL);
6248
6249                 instr->type = INSTR_METER_IMI;
6250                 if (length[0] == 'h')
6251                         instr->type = INSTR_METER_IHI;
6252
6253                 instr->meter.metarray_id = m->id;
6254
6255                 instr->meter.idx_val = idx_val;
6256
6257                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6258                 instr->meter.length.n_bits = flength->n_bits;
6259                 instr->meter.length.offset = flength->offset / 8;
6260
6261                 instr->meter.color_in_val = color_in_val;
6262
6263                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6264                 instr->meter.color_out.n_bits = fcout->n_bits;
6265                 instr->meter.color_out.offset = fcout->offset / 8;
6266
6267                 return 0;
6268         }
6269
6270         CHECK(0, EINVAL);
6271 }
6272
6273 static inline struct meter *
6274 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6275 {
6276         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6277
6278         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6279         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6280         uint64_t idx64 = *idx64_ptr;
6281         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
6282         uint64_t idx = idx64 & idx64_mask & r->size_mask;
6283
6284         return &r->metarray[idx];
6285 }
6286
6287 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6288
6289 static inline struct meter *
6290 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6291 {
6292         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6293
6294         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6295         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6296         uint64_t idx64 = *idx64_ptr;
6297         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
6298
6299         return &r->metarray[idx];
6300 }
6301
6302 #else
6303
6304 #define instr_meter_idx_nbo instr_meter_idx_hbo
6305
6306 #endif
6307
6308 static inline struct meter *
6309 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
6310 {
6311         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6312
6313         uint64_t idx =  ip->meter.idx_val & r->size_mask;
6314
6315         return &r->metarray[idx];
6316 }
6317
6318 static inline uint32_t
6319 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
6320 {
6321         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6322         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6323         uint64_t src64 = *src64_ptr;
6324         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
6325         uint64_t src = src64 & src64_mask;
6326
6327         return (uint32_t)src;
6328 }
6329
6330 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6331
6332 static inline uint32_t
6333 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
6334 {
6335         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6336         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6337         uint64_t src64 = *src64_ptr;
6338         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
6339
6340         return (uint32_t)src;
6341 }
6342
6343 #else
6344
6345 #define instr_meter_length_nbo instr_meter_length_hbo
6346
6347 #endif
6348
6349 static inline enum rte_color
6350 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
6351 {
6352         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
6353         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
6354         uint64_t src64 = *src64_ptr;
6355         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
6356         uint64_t src = src64 & src64_mask;
6357
6358         return (enum rte_color)src;
6359 }
6360
6361 static inline void
6362 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
6363 {
6364         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
6365         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
6366         uint64_t dst64 = *dst64_ptr;
6367         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
6368
6369         uint64_t src = (uint64_t)color_out;
6370
6371         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6372 }
6373
6374 static inline void
6375 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
6376 {
6377         struct thread *t = &p->threads[p->thread_id];
6378         struct instruction *ip = t->ip;
6379         struct meter *m;
6380
6381         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
6382
6383         /* Structs. */
6384         m = instr_meter_idx_nbo(p, t, ip);
6385         rte_prefetch0(m);
6386
6387         /* Thread. */
6388         thread_ip_inc(p);
6389 }
6390
6391 static inline void
6392 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
6393 {
6394         struct thread *t = &p->threads[p->thread_id];
6395         struct instruction *ip = t->ip;
6396         struct meter *m;
6397
6398         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
6399
6400         /* Structs. */
6401         m = instr_meter_idx_hbo(p, t, ip);
6402         rte_prefetch0(m);
6403
6404         /* Thread. */
6405         thread_ip_inc(p);
6406 }
6407
6408 static inline void
6409 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
6410 {
6411         struct thread *t = &p->threads[p->thread_id];
6412         struct instruction *ip = t->ip;
6413         struct meter *m;
6414
6415         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
6416
6417         /* Structs. */
6418         m = instr_meter_idx_imm(p, ip);
6419         rte_prefetch0(m);
6420
6421         /* Thread. */
6422         thread_ip_inc(p);
6423 }
6424
6425 static inline void
6426 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
6427 {
6428         struct thread *t = &p->threads[p->thread_id];
6429         struct instruction *ip = t->ip;
6430         struct meter *m;
6431         uint64_t time, n_pkts, n_bytes;
6432         uint32_t length;
6433         enum rte_color color_in, color_out;
6434
6435         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
6436
6437         /* Structs. */
6438         m = instr_meter_idx_nbo(p, t, ip);
6439         rte_prefetch0(m->n_pkts);
6440         time = rte_get_tsc_cycles();
6441         length = instr_meter_length_nbo(t, ip);
6442         color_in = instr_meter_color_in_hbo(t, ip);
6443
6444         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6445                 &m->profile->profile,
6446                 time,
6447                 length,
6448                 color_in);
6449
6450         color_out &= m->color_mask;
6451
6452         n_pkts = m->n_pkts[color_out];
6453         n_bytes = m->n_bytes[color_out];
6454
6455         instr_meter_color_out_hbo_set(t, ip, color_out);
6456
6457         m->n_pkts[color_out] = n_pkts + 1;
6458         m->n_bytes[color_out] = n_bytes + length;
6459
6460         /* Thread. */
6461         thread_ip_inc(p);
6462 }
6463
6464 static inline void
6465 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
6466 {
6467         struct thread *t = &p->threads[p->thread_id];
6468         struct instruction *ip = t->ip;
6469         struct meter *m;
6470         uint64_t time, n_pkts, n_bytes;
6471         uint32_t length;
6472         enum rte_color color_in, color_out;
6473
6474         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
6475
6476         /* Structs. */
6477         m = instr_meter_idx_nbo(p, t, ip);
6478         rte_prefetch0(m->n_pkts);
6479         time = rte_get_tsc_cycles();
6480         length = instr_meter_length_nbo(t, ip);
6481         color_in = (enum rte_color)ip->meter.color_in_val;
6482
6483         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6484                 &m->profile->profile,
6485                 time,
6486                 length,
6487                 color_in);
6488
6489         color_out &= m->color_mask;
6490
6491         n_pkts = m->n_pkts[color_out];
6492         n_bytes = m->n_bytes[color_out];
6493
6494         instr_meter_color_out_hbo_set(t, ip, color_out);
6495
6496         m->n_pkts[color_out] = n_pkts + 1;
6497         m->n_bytes[color_out] = n_bytes + length;
6498
6499         /* Thread. */
6500         thread_ip_inc(p);
6501 }
6502
6503 static inline void
6504 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
6505 {
6506         struct thread *t = &p->threads[p->thread_id];
6507         struct instruction *ip = t->ip;
6508         struct meter *m;
6509         uint64_t time, n_pkts, n_bytes;
6510         uint32_t length;
6511         enum rte_color color_in, color_out;
6512
6513         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
6514
6515         /* Structs. */
6516         m = instr_meter_idx_nbo(p, t, ip);
6517         rte_prefetch0(m->n_pkts);
6518         time = rte_get_tsc_cycles();
6519         length = instr_meter_length_hbo(t, ip);
6520         color_in = instr_meter_color_in_hbo(t, ip);
6521
6522         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6523                 &m->profile->profile,
6524                 time,
6525                 length,
6526                 color_in);
6527
6528         color_out &= m->color_mask;
6529
6530         n_pkts = m->n_pkts[color_out];
6531         n_bytes = m->n_bytes[color_out];
6532
6533         instr_meter_color_out_hbo_set(t, ip, color_out);
6534
6535         m->n_pkts[color_out] = n_pkts + 1;
6536         m->n_bytes[color_out] = n_bytes + length;
6537
6538         /* Thread. */
6539         thread_ip_inc(p);
6540 }
6541 static inline void
6542 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
6543 {
6544         struct thread *t = &p->threads[p->thread_id];
6545         struct instruction *ip = t->ip;
6546         struct meter *m;
6547         uint64_t time, n_pkts, n_bytes;
6548         uint32_t length;
6549         enum rte_color color_in, color_out;
6550
6551         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
6552
6553         /* Structs. */
6554         m = instr_meter_idx_nbo(p, t, ip);
6555         rte_prefetch0(m->n_pkts);
6556         time = rte_get_tsc_cycles();
6557         length = instr_meter_length_hbo(t, ip);
6558         color_in = (enum rte_color)ip->meter.color_in_val;
6559
6560         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6561                 &m->profile->profile,
6562                 time,
6563                 length,
6564                 color_in);
6565
6566         color_out &= m->color_mask;
6567
6568         n_pkts = m->n_pkts[color_out];
6569         n_bytes = m->n_bytes[color_out];
6570
6571         instr_meter_color_out_hbo_set(t, ip, color_out);
6572
6573         m->n_pkts[color_out] = n_pkts + 1;
6574         m->n_bytes[color_out] = n_bytes + length;
6575
6576         /* Thread. */
6577         thread_ip_inc(p);
6578 }
6579
6580 static inline void
6581 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
6582 {
6583         struct thread *t = &p->threads[p->thread_id];
6584         struct instruction *ip = t->ip;
6585         struct meter *m;
6586         uint64_t time, n_pkts, n_bytes;
6587         uint32_t length;
6588         enum rte_color color_in, color_out;
6589
6590         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
6591
6592         /* Structs. */
6593         m = instr_meter_idx_hbo(p, t, ip);
6594         rte_prefetch0(m->n_pkts);
6595         time = rte_get_tsc_cycles();
6596         length = instr_meter_length_nbo(t, ip);
6597         color_in = instr_meter_color_in_hbo(t, ip);
6598
6599         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6600                 &m->profile->profile,
6601                 time,
6602                 length,
6603                 color_in);
6604
6605         color_out &= m->color_mask;
6606
6607         n_pkts = m->n_pkts[color_out];
6608         n_bytes = m->n_bytes[color_out];
6609
6610         instr_meter_color_out_hbo_set(t, ip, color_out);
6611
6612         m->n_pkts[color_out] = n_pkts + 1;
6613         m->n_bytes[color_out] = n_bytes + length;
6614
6615         /* Thread. */
6616         thread_ip_inc(p);
6617 }
6618
6619 static inline void
6620 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
6621 {
6622         struct thread *t = &p->threads[p->thread_id];
6623         struct instruction *ip = t->ip;
6624         struct meter *m;
6625         uint64_t time, n_pkts, n_bytes;
6626         uint32_t length;
6627         enum rte_color color_in, color_out;
6628
6629         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
6630
6631         /* Structs. */
6632         m = instr_meter_idx_hbo(p, t, ip);
6633         rte_prefetch0(m->n_pkts);
6634         time = rte_get_tsc_cycles();
6635         length = instr_meter_length_nbo(t, ip);
6636         color_in = (enum rte_color)ip->meter.color_in_val;
6637
6638         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6639                 &m->profile->profile,
6640                 time,
6641                 length,
6642                 color_in);
6643
6644         color_out &= m->color_mask;
6645
6646         n_pkts = m->n_pkts[color_out];
6647         n_bytes = m->n_bytes[color_out];
6648
6649         instr_meter_color_out_hbo_set(t, ip, color_out);
6650
6651         m->n_pkts[color_out] = n_pkts + 1;
6652         m->n_bytes[color_out] = n_bytes + length;
6653
6654         /* Thread. */
6655         thread_ip_inc(p);
6656 }
6657
6658 static inline void
6659 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
6660 {
6661         struct thread *t = &p->threads[p->thread_id];
6662         struct instruction *ip = t->ip;
6663         struct meter *m;
6664         uint64_t time, n_pkts, n_bytes;
6665         uint32_t length;
6666         enum rte_color color_in, color_out;
6667
6668         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
6669
6670         /* Structs. */
6671         m = instr_meter_idx_hbo(p, t, ip);
6672         rte_prefetch0(m->n_pkts);
6673         time = rte_get_tsc_cycles();
6674         length = instr_meter_length_hbo(t, ip);
6675         color_in = instr_meter_color_in_hbo(t, ip);
6676
6677         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6678                 &m->profile->profile,
6679                 time,
6680                 length,
6681                 color_in);
6682
6683         color_out &= m->color_mask;
6684
6685         n_pkts = m->n_pkts[color_out];
6686         n_bytes = m->n_bytes[color_out];
6687
6688         instr_meter_color_out_hbo_set(t, ip, color_out);
6689
6690         m->n_pkts[color_out] = n_pkts + 1;
6691         m->n_bytes[color_out] = n_bytes + length;
6692
6693         /* Thread. */
6694         thread_ip_inc(p);
6695 }
6696
6697 static inline void
6698 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
6699 {
6700         struct thread *t = &p->threads[p->thread_id];
6701         struct instruction *ip = t->ip;
6702         struct meter *m;
6703         uint64_t time, n_pkts, n_bytes;
6704         uint32_t length;
6705         enum rte_color color_in, color_out;
6706
6707         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
6708
6709         /* Structs. */
6710         m = instr_meter_idx_hbo(p, t, ip);
6711         rte_prefetch0(m->n_pkts);
6712         time = rte_get_tsc_cycles();
6713         length = instr_meter_length_hbo(t, ip);
6714         color_in = (enum rte_color)ip->meter.color_in_val;
6715
6716         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6717                 &m->profile->profile,
6718                 time,
6719                 length,
6720                 color_in);
6721
6722         color_out &= m->color_mask;
6723
6724         n_pkts = m->n_pkts[color_out];
6725         n_bytes = m->n_bytes[color_out];
6726
6727         instr_meter_color_out_hbo_set(t, ip, color_out);
6728
6729         m->n_pkts[color_out] = n_pkts + 1;
6730         m->n_bytes[color_out] = n_bytes + length;
6731
6732         /* Thread. */
6733         thread_ip_inc(p);
6734 }
6735
6736 static inline void
6737 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
6738 {
6739         struct thread *t = &p->threads[p->thread_id];
6740         struct instruction *ip = t->ip;
6741         struct meter *m;
6742         uint64_t time, n_pkts, n_bytes;
6743         uint32_t length;
6744         enum rte_color color_in, color_out;
6745
6746         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
6747
6748         /* Structs. */
6749         m = instr_meter_idx_imm(p, ip);
6750         rte_prefetch0(m->n_pkts);
6751         time = rte_get_tsc_cycles();
6752         length = instr_meter_length_nbo(t, ip);
6753         color_in = instr_meter_color_in_hbo(t, ip);
6754
6755         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6756                 &m->profile->profile,
6757                 time,
6758                 length,
6759                 color_in);
6760
6761         color_out &= m->color_mask;
6762
6763         n_pkts = m->n_pkts[color_out];
6764         n_bytes = m->n_bytes[color_out];
6765
6766         instr_meter_color_out_hbo_set(t, ip, color_out);
6767
6768         m->n_pkts[color_out] = n_pkts + 1;
6769         m->n_bytes[color_out] = n_bytes + length;
6770
6771         /* Thread. */
6772         thread_ip_inc(p);
6773 }
6774
6775 static inline void
6776 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
6777 {
6778         struct thread *t = &p->threads[p->thread_id];
6779         struct instruction *ip = t->ip;
6780         struct meter *m;
6781         uint64_t time, n_pkts, n_bytes;
6782         uint32_t length;
6783         enum rte_color color_in, color_out;
6784
6785         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
6786
6787         /* Structs. */
6788         m = instr_meter_idx_imm(p, ip);
6789         rte_prefetch0(m->n_pkts);
6790         time = rte_get_tsc_cycles();
6791         length = instr_meter_length_nbo(t, ip);
6792         color_in = (enum rte_color)ip->meter.color_in_val;
6793
6794         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6795                 &m->profile->profile,
6796                 time,
6797                 length,
6798                 color_in);
6799
6800         color_out &= m->color_mask;
6801
6802         n_pkts = m->n_pkts[color_out];
6803         n_bytes = m->n_bytes[color_out];
6804
6805         instr_meter_color_out_hbo_set(t, ip, color_out);
6806
6807         m->n_pkts[color_out] = n_pkts + 1;
6808         m->n_bytes[color_out] = n_bytes + length;
6809
6810         /* Thread. */
6811         thread_ip_inc(p);
6812 }
6813
6814 static inline void
6815 instr_meter_imm_exec(struct rte_swx_pipeline *p)
6816 {
6817         struct thread *t = &p->threads[p->thread_id];
6818         struct instruction *ip = t->ip;
6819         struct meter *m;
6820         uint64_t time, n_pkts, n_bytes;
6821         uint32_t length;
6822         enum rte_color color_in, color_out;
6823
6824         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
6825
6826         /* Structs. */
6827         m = instr_meter_idx_imm(p, ip);
6828         rte_prefetch0(m->n_pkts);
6829         time = rte_get_tsc_cycles();
6830         length = instr_meter_length_hbo(t, ip);
6831         color_in = instr_meter_color_in_hbo(t, ip);
6832
6833         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6834                 &m->profile->profile,
6835                 time,
6836                 length,
6837                 color_in);
6838
6839         color_out &= m->color_mask;
6840
6841         n_pkts = m->n_pkts[color_out];
6842         n_bytes = m->n_bytes[color_out];
6843
6844         instr_meter_color_out_hbo_set(t, ip, color_out);
6845
6846         m->n_pkts[color_out] = n_pkts + 1;
6847         m->n_bytes[color_out] = n_bytes + length;
6848
6849         /* Thread. */
6850         thread_ip_inc(p);
6851 }
6852 static inline void
6853 instr_meter_imi_exec(struct rte_swx_pipeline *p)
6854 {
6855         struct thread *t = &p->threads[p->thread_id];
6856         struct instruction *ip = t->ip;
6857         struct meter *m;
6858         uint64_t time, n_pkts, n_bytes;
6859         uint32_t length;
6860         enum rte_color color_in, color_out;
6861
6862         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
6863
6864         /* Structs. */
6865         m = instr_meter_idx_imm(p, ip);
6866         rte_prefetch0(m->n_pkts);
6867         time = rte_get_tsc_cycles();
6868         length = instr_meter_length_hbo(t, ip);
6869         color_in = (enum rte_color)ip->meter.color_in_val;
6870
6871         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6872                 &m->profile->profile,
6873                 time,
6874                 length,
6875                 color_in);
6876
6877         color_out &= m->color_mask;
6878
6879         n_pkts = m->n_pkts[color_out];
6880         n_bytes = m->n_bytes[color_out];
6881
6882         instr_meter_color_out_hbo_set(t, ip, color_out);
6883
6884         m->n_pkts[color_out] = n_pkts + 1;
6885         m->n_bytes[color_out] = n_bytes + length;
6886
6887         /* Thread. */
6888         thread_ip_inc(p);
6889 }
6890
6891 /*
6892  * jmp.
6893  */
6894 static struct action *
6895 action_find(struct rte_swx_pipeline *p, const char *name);
6896
6897 static int
6898 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
6899                     struct action *action __rte_unused,
6900                     char **tokens,
6901                     int n_tokens,
6902                     struct instruction *instr,
6903                     struct instruction_data *data)
6904 {
6905         CHECK(n_tokens == 2, EINVAL);
6906
6907         strcpy(data->jmp_label, tokens[1]);
6908
6909         instr->type = INSTR_JMP;
6910         instr->jmp.ip = NULL; /* Resolved later. */
6911         return 0;
6912 }
6913
6914 static int
6915 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
6916                           struct action *action __rte_unused,
6917                           char **tokens,
6918                           int n_tokens,
6919                           struct instruction *instr,
6920                           struct instruction_data *data)
6921 {
6922         struct header *h;
6923
6924         CHECK(n_tokens == 3, EINVAL);
6925
6926         strcpy(data->jmp_label, tokens[1]);
6927
6928         h = header_parse(p, tokens[2]);
6929         CHECK(h, EINVAL);
6930
6931         instr->type = INSTR_JMP_VALID;
6932         instr->jmp.ip = NULL; /* Resolved later. */
6933         instr->jmp.header_id = h->id;
6934         return 0;
6935 }
6936
6937 static int
6938 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
6939                             struct action *action __rte_unused,
6940                             char **tokens,
6941                             int n_tokens,
6942                             struct instruction *instr,
6943                             struct instruction_data *data)
6944 {
6945         struct header *h;
6946
6947         CHECK(n_tokens == 3, EINVAL);
6948
6949         strcpy(data->jmp_label, tokens[1]);
6950
6951         h = header_parse(p, tokens[2]);
6952         CHECK(h, EINVAL);
6953
6954         instr->type = INSTR_JMP_INVALID;
6955         instr->jmp.ip = NULL; /* Resolved later. */
6956         instr->jmp.header_id = h->id;
6957         return 0;
6958 }
6959
6960 static int
6961 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
6962                         struct action *action,
6963                         char **tokens,
6964                         int n_tokens,
6965                         struct instruction *instr,
6966                         struct instruction_data *data)
6967 {
6968         CHECK(!action, EINVAL);
6969         CHECK(n_tokens == 2, EINVAL);
6970
6971         strcpy(data->jmp_label, tokens[1]);
6972
6973         instr->type = INSTR_JMP_HIT;
6974         instr->jmp.ip = NULL; /* Resolved later. */
6975         return 0;
6976 }
6977
6978 static int
6979 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
6980                          struct action *action,
6981                          char **tokens,
6982                          int n_tokens,
6983                          struct instruction *instr,
6984                          struct instruction_data *data)
6985 {
6986         CHECK(!action, EINVAL);
6987         CHECK(n_tokens == 2, EINVAL);
6988
6989         strcpy(data->jmp_label, tokens[1]);
6990
6991         instr->type = INSTR_JMP_MISS;
6992         instr->jmp.ip = NULL; /* Resolved later. */
6993         return 0;
6994 }
6995
6996 static int
6997 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
6998                                struct action *action,
6999                                char **tokens,
7000                                int n_tokens,
7001                                struct instruction *instr,
7002                                struct instruction_data *data)
7003 {
7004         struct action *a;
7005
7006         CHECK(!action, EINVAL);
7007         CHECK(n_tokens == 3, EINVAL);
7008
7009         strcpy(data->jmp_label, tokens[1]);
7010
7011         a = action_find(p, tokens[2]);
7012         CHECK(a, EINVAL);
7013
7014         instr->type = INSTR_JMP_ACTION_HIT;
7015         instr->jmp.ip = NULL; /* Resolved later. */
7016         instr->jmp.action_id = a->id;
7017         return 0;
7018 }
7019
7020 static int
7021 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
7022                                 struct action *action,
7023                                 char **tokens,
7024                                 int n_tokens,
7025                                 struct instruction *instr,
7026                                 struct instruction_data *data)
7027 {
7028         struct action *a;
7029
7030         CHECK(!action, EINVAL);
7031         CHECK(n_tokens == 3, EINVAL);
7032
7033         strcpy(data->jmp_label, tokens[1]);
7034
7035         a = action_find(p, tokens[2]);
7036         CHECK(a, EINVAL);
7037
7038         instr->type = INSTR_JMP_ACTION_MISS;
7039         instr->jmp.ip = NULL; /* Resolved later. */
7040         instr->jmp.action_id = a->id;
7041         return 0;
7042 }
7043
7044 static int
7045 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
7046                        struct action *action,
7047                        char **tokens,
7048                        int n_tokens,
7049                        struct instruction *instr,
7050                        struct instruction_data *data)
7051 {
7052         char *a = tokens[2], *b = tokens[3];
7053         struct field *fa, *fb;
7054         uint64_t b_val;
7055         uint32_t a_struct_id, b_struct_id;
7056
7057         CHECK(n_tokens == 4, EINVAL);
7058
7059         strcpy(data->jmp_label, tokens[1]);
7060
7061         fa = struct_field_parse(p, action, a, &a_struct_id);
7062         CHECK(fa, EINVAL);
7063
7064         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
7065         fb = struct_field_parse(p, action, b, &b_struct_id);
7066         if (fb) {
7067                 instr->type = INSTR_JMP_EQ;
7068                 if (a[0] != 'h' && b[0] == 'h')
7069                         instr->type = INSTR_JMP_EQ_MH;
7070                 if (a[0] == 'h' && b[0] != 'h')
7071                         instr->type = INSTR_JMP_EQ_HM;
7072                 if (a[0] == 'h' && b[0] == 'h')
7073                         instr->type = INSTR_JMP_EQ_HH;
7074                 instr->jmp.ip = NULL; /* Resolved later. */
7075
7076                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7077                 instr->jmp.a.n_bits = fa->n_bits;
7078                 instr->jmp.a.offset = fa->offset / 8;
7079                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7080                 instr->jmp.b.n_bits = fb->n_bits;
7081                 instr->jmp.b.offset = fb->offset / 8;
7082                 return 0;
7083         }
7084
7085         /* JMP_EQ_I. */
7086         b_val = strtoull(b, &b, 0);
7087         CHECK(!b[0], EINVAL);
7088
7089         if (a[0] == 'h')
7090                 b_val = hton64(b_val) >> (64 - fa->n_bits);
7091
7092         instr->type = INSTR_JMP_EQ_I;
7093         instr->jmp.ip = NULL; /* Resolved later. */
7094         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7095         instr->jmp.a.n_bits = fa->n_bits;
7096         instr->jmp.a.offset = fa->offset / 8;
7097         instr->jmp.b_val = b_val;
7098         return 0;
7099 }
7100
7101 static int
7102 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
7103                         struct action *action,
7104                         char **tokens,
7105                         int n_tokens,
7106                         struct instruction *instr,
7107                         struct instruction_data *data)
7108 {
7109         char *a = tokens[2], *b = tokens[3];
7110         struct field *fa, *fb;
7111         uint64_t b_val;
7112         uint32_t a_struct_id, b_struct_id;
7113
7114         CHECK(n_tokens == 4, EINVAL);
7115
7116         strcpy(data->jmp_label, tokens[1]);
7117
7118         fa = struct_field_parse(p, action, a, &a_struct_id);
7119         CHECK(fa, EINVAL);
7120
7121         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
7122         fb = struct_field_parse(p, action, b, &b_struct_id);
7123         if (fb) {
7124                 instr->type = INSTR_JMP_NEQ;
7125                 if (a[0] != 'h' && b[0] == 'h')
7126                         instr->type = INSTR_JMP_NEQ_MH;
7127                 if (a[0] == 'h' && b[0] != 'h')
7128                         instr->type = INSTR_JMP_NEQ_HM;
7129                 if (a[0] == 'h' && b[0] == 'h')
7130                         instr->type = INSTR_JMP_NEQ_HH;
7131                 instr->jmp.ip = NULL; /* Resolved later. */
7132
7133                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7134                 instr->jmp.a.n_bits = fa->n_bits;
7135                 instr->jmp.a.offset = fa->offset / 8;
7136                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7137                 instr->jmp.b.n_bits = fb->n_bits;
7138                 instr->jmp.b.offset = fb->offset / 8;
7139                 return 0;
7140         }
7141
7142         /* JMP_NEQ_I. */
7143         b_val = strtoull(b, &b, 0);
7144         CHECK(!b[0], EINVAL);
7145
7146         if (a[0] == 'h')
7147                 b_val = hton64(b_val) >> (64 - fa->n_bits);
7148
7149         instr->type = INSTR_JMP_NEQ_I;
7150         instr->jmp.ip = NULL; /* Resolved later. */
7151         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7152         instr->jmp.a.n_bits = fa->n_bits;
7153         instr->jmp.a.offset = fa->offset / 8;
7154         instr->jmp.b_val = b_val;
7155         return 0;
7156 }
7157
7158 static int
7159 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
7160                        struct action *action,
7161                        char **tokens,
7162                        int n_tokens,
7163                        struct instruction *instr,
7164                        struct instruction_data *data)
7165 {
7166         char *a = tokens[2], *b = tokens[3];
7167         struct field *fa, *fb;
7168         uint64_t b_val;
7169         uint32_t a_struct_id, b_struct_id;
7170
7171         CHECK(n_tokens == 4, EINVAL);
7172
7173         strcpy(data->jmp_label, tokens[1]);
7174
7175         fa = struct_field_parse(p, action, a, &a_struct_id);
7176         CHECK(fa, EINVAL);
7177
7178         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
7179         fb = struct_field_parse(p, action, b, &b_struct_id);
7180         if (fb) {
7181                 instr->type = INSTR_JMP_LT;
7182                 if (a[0] == 'h' && b[0] != 'h')
7183                         instr->type = INSTR_JMP_LT_HM;
7184                 if (a[0] != 'h' && b[0] == 'h')
7185                         instr->type = INSTR_JMP_LT_MH;
7186                 if (a[0] == 'h' && b[0] == 'h')
7187                         instr->type = INSTR_JMP_LT_HH;
7188                 instr->jmp.ip = NULL; /* Resolved later. */
7189
7190                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7191                 instr->jmp.a.n_bits = fa->n_bits;
7192                 instr->jmp.a.offset = fa->offset / 8;
7193                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7194                 instr->jmp.b.n_bits = fb->n_bits;
7195                 instr->jmp.b.offset = fb->offset / 8;
7196                 return 0;
7197         }
7198
7199         /* JMP_LT_MI, JMP_LT_HI. */
7200         b_val = strtoull(b, &b, 0);
7201         CHECK(!b[0], EINVAL);
7202
7203         instr->type = INSTR_JMP_LT_MI;
7204         if (a[0] == 'h')
7205                 instr->type = INSTR_JMP_LT_HI;
7206         instr->jmp.ip = NULL; /* Resolved later. */
7207
7208         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7209         instr->jmp.a.n_bits = fa->n_bits;
7210         instr->jmp.a.offset = fa->offset / 8;
7211         instr->jmp.b_val = b_val;
7212         return 0;
7213 }
7214
7215 static int
7216 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
7217                        struct action *action,
7218                        char **tokens,
7219                        int n_tokens,
7220                        struct instruction *instr,
7221                        struct instruction_data *data)
7222 {
7223         char *a = tokens[2], *b = tokens[3];
7224         struct field *fa, *fb;
7225         uint64_t b_val;
7226         uint32_t a_struct_id, b_struct_id;
7227
7228         CHECK(n_tokens == 4, EINVAL);
7229
7230         strcpy(data->jmp_label, tokens[1]);
7231
7232         fa = struct_field_parse(p, action, a, &a_struct_id);
7233         CHECK(fa, EINVAL);
7234
7235         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
7236         fb = struct_field_parse(p, action, b, &b_struct_id);
7237         if (fb) {
7238                 instr->type = INSTR_JMP_GT;
7239                 if (a[0] == 'h' && b[0] != 'h')
7240                         instr->type = INSTR_JMP_GT_HM;
7241                 if (a[0] != 'h' && b[0] == 'h')
7242                         instr->type = INSTR_JMP_GT_MH;
7243                 if (a[0] == 'h' && b[0] == 'h')
7244                         instr->type = INSTR_JMP_GT_HH;
7245                 instr->jmp.ip = NULL; /* Resolved later. */
7246
7247                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7248                 instr->jmp.a.n_bits = fa->n_bits;
7249                 instr->jmp.a.offset = fa->offset / 8;
7250                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7251                 instr->jmp.b.n_bits = fb->n_bits;
7252                 instr->jmp.b.offset = fb->offset / 8;
7253                 return 0;
7254         }
7255
7256         /* JMP_GT_MI, JMP_GT_HI. */
7257         b_val = strtoull(b, &b, 0);
7258         CHECK(!b[0], EINVAL);
7259
7260         instr->type = INSTR_JMP_GT_MI;
7261         if (a[0] == 'h')
7262                 instr->type = INSTR_JMP_GT_HI;
7263         instr->jmp.ip = NULL; /* Resolved later. */
7264
7265         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7266         instr->jmp.a.n_bits = fa->n_bits;
7267         instr->jmp.a.offset = fa->offset / 8;
7268         instr->jmp.b_val = b_val;
7269         return 0;
7270 }
7271
7272 static inline void
7273 instr_jmp_exec(struct rte_swx_pipeline *p)
7274 {
7275         struct thread *t = &p->threads[p->thread_id];
7276         struct instruction *ip = t->ip;
7277
7278         TRACE("[Thread %2u] jmp\n", p->thread_id);
7279
7280         thread_ip_set(t, ip->jmp.ip);
7281 }
7282
7283 static inline void
7284 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
7285 {
7286         struct thread *t = &p->threads[p->thread_id];
7287         struct instruction *ip = t->ip;
7288         uint32_t header_id = ip->jmp.header_id;
7289
7290         TRACE("[Thread %2u] jmpv\n", p->thread_id);
7291
7292         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
7293 }
7294
7295 static inline void
7296 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
7297 {
7298         struct thread *t = &p->threads[p->thread_id];
7299         struct instruction *ip = t->ip;
7300         uint32_t header_id = ip->jmp.header_id;
7301
7302         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
7303
7304         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
7305 }
7306
7307 static inline void
7308 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
7309 {
7310         struct thread *t = &p->threads[p->thread_id];
7311         struct instruction *ip = t->ip;
7312         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
7313
7314         TRACE("[Thread %2u] jmph\n", p->thread_id);
7315
7316         t->ip = ip_next[t->hit];
7317 }
7318
7319 static inline void
7320 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
7321 {
7322         struct thread *t = &p->threads[p->thread_id];
7323         struct instruction *ip = t->ip;
7324         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
7325
7326         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
7327
7328         t->ip = ip_next[t->hit];
7329 }
7330
7331 static inline void
7332 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
7333 {
7334         struct thread *t = &p->threads[p->thread_id];
7335         struct instruction *ip = t->ip;
7336
7337         TRACE("[Thread %2u] jmpa\n", p->thread_id);
7338
7339         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
7340 }
7341
7342 static inline void
7343 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
7344 {
7345         struct thread *t = &p->threads[p->thread_id];
7346         struct instruction *ip = t->ip;
7347
7348         TRACE("[Thread %2u] jmpna\n", p->thread_id);
7349
7350         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
7351 }
7352
7353 static inline void
7354 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
7355 {
7356         struct thread *t = &p->threads[p->thread_id];
7357         struct instruction *ip = t->ip;
7358
7359         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
7360
7361         JMP_CMP(t, ip, ==);
7362 }
7363
7364 static inline void
7365 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
7366 {
7367         struct thread *t = &p->threads[p->thread_id];
7368         struct instruction *ip = t->ip;
7369
7370         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
7371
7372         JMP_CMP_MH(t, ip, ==);
7373 }
7374
7375 static inline void
7376 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
7377 {
7378         struct thread *t = &p->threads[p->thread_id];
7379         struct instruction *ip = t->ip;
7380
7381         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
7382
7383         JMP_CMP_HM(t, ip, ==);
7384 }
7385
7386 static inline void
7387 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
7388 {
7389         struct thread *t = &p->threads[p->thread_id];
7390         struct instruction *ip = t->ip;
7391
7392         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
7393
7394         JMP_CMP_HH_FAST(t, ip, ==);
7395 }
7396
7397 static inline void
7398 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
7399 {
7400         struct thread *t = &p->threads[p->thread_id];
7401         struct instruction *ip = t->ip;
7402
7403         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
7404
7405         JMP_CMP_I(t, ip, ==);
7406 }
7407
7408 static inline void
7409 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
7410 {
7411         struct thread *t = &p->threads[p->thread_id];
7412         struct instruction *ip = t->ip;
7413
7414         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
7415
7416         JMP_CMP(t, ip, !=);
7417 }
7418
7419 static inline void
7420 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
7421 {
7422         struct thread *t = &p->threads[p->thread_id];
7423         struct instruction *ip = t->ip;
7424
7425         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
7426
7427         JMP_CMP_MH(t, ip, !=);
7428 }
7429
7430 static inline void
7431 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
7432 {
7433         struct thread *t = &p->threads[p->thread_id];
7434         struct instruction *ip = t->ip;
7435
7436         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
7437
7438         JMP_CMP_HM(t, ip, !=);
7439 }
7440
7441 static inline void
7442 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
7443 {
7444         struct thread *t = &p->threads[p->thread_id];
7445         struct instruction *ip = t->ip;
7446
7447         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
7448
7449         JMP_CMP_HH_FAST(t, ip, !=);
7450 }
7451
7452 static inline void
7453 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
7454 {
7455         struct thread *t = &p->threads[p->thread_id];
7456         struct instruction *ip = t->ip;
7457
7458         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
7459
7460         JMP_CMP_I(t, ip, !=);
7461 }
7462
7463 static inline void
7464 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
7465 {
7466         struct thread *t = &p->threads[p->thread_id];
7467         struct instruction *ip = t->ip;
7468
7469         TRACE("[Thread %2u] jmplt\n", p->thread_id);
7470
7471         JMP_CMP(t, ip, <);
7472 }
7473
7474 static inline void
7475 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
7476 {
7477         struct thread *t = &p->threads[p->thread_id];
7478         struct instruction *ip = t->ip;
7479
7480         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
7481
7482         JMP_CMP_MH(t, ip, <);
7483 }
7484
7485 static inline void
7486 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
7487 {
7488         struct thread *t = &p->threads[p->thread_id];
7489         struct instruction *ip = t->ip;
7490
7491         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
7492
7493         JMP_CMP_HM(t, ip, <);
7494 }
7495
7496 static inline void
7497 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
7498 {
7499         struct thread *t = &p->threads[p->thread_id];
7500         struct instruction *ip = t->ip;
7501
7502         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
7503
7504         JMP_CMP_HH(t, ip, <);
7505 }
7506
7507 static inline void
7508 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
7509 {
7510         struct thread *t = &p->threads[p->thread_id];
7511         struct instruction *ip = t->ip;
7512
7513         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
7514
7515         JMP_CMP_MI(t, ip, <);
7516 }
7517
7518 static inline void
7519 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
7520 {
7521         struct thread *t = &p->threads[p->thread_id];
7522         struct instruction *ip = t->ip;
7523
7524         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
7525
7526         JMP_CMP_HI(t, ip, <);
7527 }
7528
7529 static inline void
7530 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
7531 {
7532         struct thread *t = &p->threads[p->thread_id];
7533         struct instruction *ip = t->ip;
7534
7535         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
7536
7537         JMP_CMP(t, ip, >);
7538 }
7539
7540 static inline void
7541 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
7542 {
7543         struct thread *t = &p->threads[p->thread_id];
7544         struct instruction *ip = t->ip;
7545
7546         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
7547
7548         JMP_CMP_MH(t, ip, >);
7549 }
7550
7551 static inline void
7552 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
7553 {
7554         struct thread *t = &p->threads[p->thread_id];
7555         struct instruction *ip = t->ip;
7556
7557         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
7558
7559         JMP_CMP_HM(t, ip, >);
7560 }
7561
7562 static inline void
7563 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
7564 {
7565         struct thread *t = &p->threads[p->thread_id];
7566         struct instruction *ip = t->ip;
7567
7568         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
7569
7570         JMP_CMP_HH(t, ip, >);
7571 }
7572
7573 static inline void
7574 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
7575 {
7576         struct thread *t = &p->threads[p->thread_id];
7577         struct instruction *ip = t->ip;
7578
7579         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
7580
7581         JMP_CMP_MI(t, ip, >);
7582 }
7583
7584 static inline void
7585 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
7586 {
7587         struct thread *t = &p->threads[p->thread_id];
7588         struct instruction *ip = t->ip;
7589
7590         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
7591
7592         JMP_CMP_HI(t, ip, >);
7593 }
7594
7595 /*
7596  * return.
7597  */
7598 static int
7599 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
7600                        struct action *action,
7601                        char **tokens __rte_unused,
7602                        int n_tokens,
7603                        struct instruction *instr,
7604                        struct instruction_data *data __rte_unused)
7605 {
7606         CHECK(action, EINVAL);
7607         CHECK(n_tokens == 1, EINVAL);
7608
7609         instr->type = INSTR_RETURN;
7610         return 0;
7611 }
7612
7613 static inline void
7614 instr_return_exec(struct rte_swx_pipeline *p)
7615 {
7616         struct thread *t = &p->threads[p->thread_id];
7617
7618         TRACE("[Thread %2u] return\n", p->thread_id);
7619
7620         t->ip = t->ret;
7621 }
7622
7623 static int
7624 instr_translate(struct rte_swx_pipeline *p,
7625                 struct action *action,
7626                 char *string,
7627                 struct instruction *instr,
7628                 struct instruction_data *data)
7629 {
7630         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
7631         int n_tokens = 0, tpos = 0;
7632
7633         /* Parse the instruction string into tokens. */
7634         for ( ; ; ) {
7635                 char *token;
7636
7637                 token = strtok_r(string, " \t\v", &string);
7638                 if (!token)
7639                         break;
7640
7641                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
7642                 CHECK_NAME(token, EINVAL);
7643
7644                 tokens[n_tokens] = token;
7645                 n_tokens++;
7646         }
7647
7648         CHECK(n_tokens, EINVAL);
7649
7650         /* Handle the optional instruction label. */
7651         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
7652                 strcpy(data->label, tokens[0]);
7653
7654                 tpos += 2;
7655                 CHECK(n_tokens - tpos, EINVAL);
7656         }
7657
7658         /* Identify the instruction type. */
7659         if (!strcmp(tokens[tpos], "rx"))
7660                 return instr_rx_translate(p,
7661                                           action,
7662                                           &tokens[tpos],
7663                                           n_tokens - tpos,
7664                                           instr,
7665                                           data);
7666
7667         if (!strcmp(tokens[tpos], "tx"))
7668                 return instr_tx_translate(p,
7669                                           action,
7670                                           &tokens[tpos],
7671                                           n_tokens - tpos,
7672                                           instr,
7673                                           data);
7674
7675         if (!strcmp(tokens[tpos], "drop"))
7676                 return instr_drop_translate(p,
7677                                             action,
7678                                             &tokens[tpos],
7679                                             n_tokens - tpos,
7680                                             instr,
7681                                             data);
7682
7683         if (!strcmp(tokens[tpos], "extract"))
7684                 return instr_hdr_extract_translate(p,
7685                                                    action,
7686                                                    &tokens[tpos],
7687                                                    n_tokens - tpos,
7688                                                    instr,
7689                                                    data);
7690
7691         if (!strcmp(tokens[tpos], "emit"))
7692                 return instr_hdr_emit_translate(p,
7693                                                 action,
7694                                                 &tokens[tpos],
7695                                                 n_tokens - tpos,
7696                                                 instr,
7697                                                 data);
7698
7699         if (!strcmp(tokens[tpos], "validate"))
7700                 return instr_hdr_validate_translate(p,
7701                                                     action,
7702                                                     &tokens[tpos],
7703                                                     n_tokens - tpos,
7704                                                     instr,
7705                                                     data);
7706
7707         if (!strcmp(tokens[tpos], "invalidate"))
7708                 return instr_hdr_invalidate_translate(p,
7709                                                       action,
7710                                                       &tokens[tpos],
7711                                                       n_tokens - tpos,
7712                                                       instr,
7713                                                       data);
7714
7715         if (!strcmp(tokens[tpos], "mov"))
7716                 return instr_mov_translate(p,
7717                                            action,
7718                                            &tokens[tpos],
7719                                            n_tokens - tpos,
7720                                            instr,
7721                                            data);
7722
7723         if (!strcmp(tokens[tpos], "dma"))
7724                 return instr_dma_translate(p,
7725                                            action,
7726                                            &tokens[tpos],
7727                                            n_tokens - tpos,
7728                                            instr,
7729                                            data);
7730
7731         if (!strcmp(tokens[tpos], "add"))
7732                 return instr_alu_add_translate(p,
7733                                                action,
7734                                                &tokens[tpos],
7735                                                n_tokens - tpos,
7736                                                instr,
7737                                                data);
7738
7739         if (!strcmp(tokens[tpos], "sub"))
7740                 return instr_alu_sub_translate(p,
7741                                                action,
7742                                                &tokens[tpos],
7743                                                n_tokens - tpos,
7744                                                instr,
7745                                                data);
7746
7747         if (!strcmp(tokens[tpos], "ckadd"))
7748                 return instr_alu_ckadd_translate(p,
7749                                                  action,
7750                                                  &tokens[tpos],
7751                                                  n_tokens - tpos,
7752                                                  instr,
7753                                                  data);
7754
7755         if (!strcmp(tokens[tpos], "cksub"))
7756                 return instr_alu_cksub_translate(p,
7757                                                  action,
7758                                                  &tokens[tpos],
7759                                                  n_tokens - tpos,
7760                                                  instr,
7761                                                  data);
7762
7763         if (!strcmp(tokens[tpos], "and"))
7764                 return instr_alu_and_translate(p,
7765                                                action,
7766                                                &tokens[tpos],
7767                                                n_tokens - tpos,
7768                                                instr,
7769                                                data);
7770
7771         if (!strcmp(tokens[tpos], "or"))
7772                 return instr_alu_or_translate(p,
7773                                               action,
7774                                               &tokens[tpos],
7775                                               n_tokens - tpos,
7776                                               instr,
7777                                               data);
7778
7779         if (!strcmp(tokens[tpos], "xor"))
7780                 return instr_alu_xor_translate(p,
7781                                                action,
7782                                                &tokens[tpos],
7783                                                n_tokens - tpos,
7784                                                instr,
7785                                                data);
7786
7787         if (!strcmp(tokens[tpos], "shl"))
7788                 return instr_alu_shl_translate(p,
7789                                                action,
7790                                                &tokens[tpos],
7791                                                n_tokens - tpos,
7792                                                instr,
7793                                                data);
7794
7795         if (!strcmp(tokens[tpos], "shr"))
7796                 return instr_alu_shr_translate(p,
7797                                                action,
7798                                                &tokens[tpos],
7799                                                n_tokens - tpos,
7800                                                instr,
7801                                                data);
7802
7803         if (!strcmp(tokens[tpos], "regprefetch"))
7804                 return instr_regprefetch_translate(p,
7805                                                    action,
7806                                                    &tokens[tpos],
7807                                                    n_tokens - tpos,
7808                                                    instr,
7809                                                    data);
7810
7811         if (!strcmp(tokens[tpos], "regrd"))
7812                 return instr_regrd_translate(p,
7813                                              action,
7814                                              &tokens[tpos],
7815                                              n_tokens - tpos,
7816                                              instr,
7817                                              data);
7818
7819         if (!strcmp(tokens[tpos], "regwr"))
7820                 return instr_regwr_translate(p,
7821                                              action,
7822                                              &tokens[tpos],
7823                                              n_tokens - tpos,
7824                                              instr,
7825                                              data);
7826
7827         if (!strcmp(tokens[tpos], "regadd"))
7828                 return instr_regadd_translate(p,
7829                                               action,
7830                                               &tokens[tpos],
7831                                               n_tokens - tpos,
7832                                               instr,
7833                                               data);
7834
7835         if (!strcmp(tokens[tpos], "metprefetch"))
7836                 return instr_metprefetch_translate(p,
7837                                                    action,
7838                                                    &tokens[tpos],
7839                                                    n_tokens - tpos,
7840                                                    instr,
7841                                                    data);
7842
7843         if (!strcmp(tokens[tpos], "meter"))
7844                 return instr_meter_translate(p,
7845                                              action,
7846                                              &tokens[tpos],
7847                                              n_tokens - tpos,
7848                                              instr,
7849                                              data);
7850
7851         if (!strcmp(tokens[tpos], "table"))
7852                 return instr_table_translate(p,
7853                                              action,
7854                                              &tokens[tpos],
7855                                              n_tokens - tpos,
7856                                              instr,
7857                                              data);
7858
7859         if (!strcmp(tokens[tpos], "extern"))
7860                 return instr_extern_translate(p,
7861                                               action,
7862                                               &tokens[tpos],
7863                                               n_tokens - tpos,
7864                                               instr,
7865                                               data);
7866
7867         if (!strcmp(tokens[tpos], "jmp"))
7868                 return instr_jmp_translate(p,
7869                                            action,
7870                                            &tokens[tpos],
7871                                            n_tokens - tpos,
7872                                            instr,
7873                                            data);
7874
7875         if (!strcmp(tokens[tpos], "jmpv"))
7876                 return instr_jmp_valid_translate(p,
7877                                                  action,
7878                                                  &tokens[tpos],
7879                                                  n_tokens - tpos,
7880                                                  instr,
7881                                                  data);
7882
7883         if (!strcmp(tokens[tpos], "jmpnv"))
7884                 return instr_jmp_invalid_translate(p,
7885                                                    action,
7886                                                    &tokens[tpos],
7887                                                    n_tokens - tpos,
7888                                                    instr,
7889                                                    data);
7890
7891         if (!strcmp(tokens[tpos], "jmph"))
7892                 return instr_jmp_hit_translate(p,
7893                                                action,
7894                                                &tokens[tpos],
7895                                                n_tokens - tpos,
7896                                                instr,
7897                                                data);
7898
7899         if (!strcmp(tokens[tpos], "jmpnh"))
7900                 return instr_jmp_miss_translate(p,
7901                                                 action,
7902                                                 &tokens[tpos],
7903                                                 n_tokens - tpos,
7904                                                 instr,
7905                                                 data);
7906
7907         if (!strcmp(tokens[tpos], "jmpa"))
7908                 return instr_jmp_action_hit_translate(p,
7909                                                       action,
7910                                                       &tokens[tpos],
7911                                                       n_tokens - tpos,
7912                                                       instr,
7913                                                       data);
7914
7915         if (!strcmp(tokens[tpos], "jmpna"))
7916                 return instr_jmp_action_miss_translate(p,
7917                                                        action,
7918                                                        &tokens[tpos],
7919                                                        n_tokens - tpos,
7920                                                        instr,
7921                                                        data);
7922
7923         if (!strcmp(tokens[tpos], "jmpeq"))
7924                 return instr_jmp_eq_translate(p,
7925                                               action,
7926                                               &tokens[tpos],
7927                                               n_tokens - tpos,
7928                                               instr,
7929                                               data);
7930
7931         if (!strcmp(tokens[tpos], "jmpneq"))
7932                 return instr_jmp_neq_translate(p,
7933                                                action,
7934                                                &tokens[tpos],
7935                                                n_tokens - tpos,
7936                                                instr,
7937                                                data);
7938
7939         if (!strcmp(tokens[tpos], "jmplt"))
7940                 return instr_jmp_lt_translate(p,
7941                                               action,
7942                                               &tokens[tpos],
7943                                               n_tokens - tpos,
7944                                               instr,
7945                                               data);
7946
7947         if (!strcmp(tokens[tpos], "jmpgt"))
7948                 return instr_jmp_gt_translate(p,
7949                                               action,
7950                                               &tokens[tpos],
7951                                               n_tokens - tpos,
7952                                               instr,
7953                                               data);
7954
7955         if (!strcmp(tokens[tpos], "return"))
7956                 return instr_return_translate(p,
7957                                               action,
7958                                               &tokens[tpos],
7959                                               n_tokens - tpos,
7960                                               instr,
7961                                               data);
7962
7963         CHECK(0, EINVAL);
7964 }
7965
7966 static struct instruction_data *
7967 label_find(struct instruction_data *data, uint32_t n, const char *label)
7968 {
7969         uint32_t i;
7970
7971         for (i = 0; i < n; i++)
7972                 if (!strcmp(label, data[i].label))
7973                         return &data[i];
7974
7975         return NULL;
7976 }
7977
7978 static uint32_t
7979 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
7980 {
7981         uint32_t count = 0, i;
7982
7983         if (!label[0])
7984                 return 0;
7985
7986         for (i = 0; i < n; i++)
7987                 if (!strcmp(label, data[i].jmp_label))
7988                         count++;
7989
7990         return count;
7991 }
7992
7993 static int
7994 instr_label_check(struct instruction_data *instruction_data,
7995                   uint32_t n_instructions)
7996 {
7997         uint32_t i;
7998
7999         /* Check that all instruction labels are unique. */
8000         for (i = 0; i < n_instructions; i++) {
8001                 struct instruction_data *data = &instruction_data[i];
8002                 char *label = data->label;
8003                 uint32_t j;
8004
8005                 if (!label[0])
8006                         continue;
8007
8008                 for (j = i + 1; j < n_instructions; j++)
8009                         CHECK(strcmp(label, data[j].label), EINVAL);
8010         }
8011
8012         /* Get users for each instruction label. */
8013         for (i = 0; i < n_instructions; i++) {
8014                 struct instruction_data *data = &instruction_data[i];
8015                 char *label = data->label;
8016
8017                 data->n_users = label_is_used(instruction_data,
8018                                               n_instructions,
8019                                               label);
8020         }
8021
8022         return 0;
8023 }
8024
8025 static int
8026 instr_jmp_resolve(struct instruction *instructions,
8027                   struct instruction_data *instruction_data,
8028                   uint32_t n_instructions)
8029 {
8030         uint32_t i;
8031
8032         for (i = 0; i < n_instructions; i++) {
8033                 struct instruction *instr = &instructions[i];
8034                 struct instruction_data *data = &instruction_data[i];
8035                 struct instruction_data *found;
8036
8037                 if (!instruction_is_jmp(instr))
8038                         continue;
8039
8040                 found = label_find(instruction_data,
8041                                    n_instructions,
8042                                    data->jmp_label);
8043                 CHECK(found, EINVAL);
8044
8045                 instr->jmp.ip = &instructions[found - instruction_data];
8046         }
8047
8048         return 0;
8049 }
8050
8051 static int
8052 instr_verify(struct rte_swx_pipeline *p __rte_unused,
8053              struct action *a,
8054              struct instruction *instr,
8055              struct instruction_data *data __rte_unused,
8056              uint32_t n_instructions)
8057 {
8058         if (!a) {
8059                 enum instruction_type type;
8060                 uint32_t i;
8061
8062                 /* Check that the first instruction is rx. */
8063                 CHECK(instr[0].type == INSTR_RX, EINVAL);
8064
8065                 /* Check that there is at least one tx instruction. */
8066                 for (i = 0; i < n_instructions; i++) {
8067                         type = instr[i].type;
8068
8069                         if (instruction_is_tx(type))
8070                                 break;
8071                 }
8072                 CHECK(i < n_instructions, EINVAL);
8073
8074                 /* Check that the last instruction is either tx or unconditional
8075                  * jump.
8076                  */
8077                 type = instr[n_instructions - 1].type;
8078                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
8079         }
8080
8081         if (a) {
8082                 enum instruction_type type;
8083                 uint32_t i;
8084
8085                 /* Check that there is at least one return or tx instruction. */
8086                 for (i = 0; i < n_instructions; i++) {
8087                         type = instr[i].type;
8088
8089                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
8090                                 break;
8091                 }
8092                 CHECK(i < n_instructions, EINVAL);
8093         }
8094
8095         return 0;
8096 }
8097
8098 static uint32_t
8099 instr_compact(struct instruction *instructions,
8100               struct instruction_data *instruction_data,
8101               uint32_t n_instructions)
8102 {
8103         uint32_t i, pos = 0;
8104
8105         /* Eliminate the invalid instructions that have been optimized out. */
8106         for (i = 0; i < n_instructions; i++) {
8107                 struct instruction *instr = &instructions[i];
8108                 struct instruction_data *data = &instruction_data[i];
8109
8110                 if (data->invalid)
8111                         continue;
8112
8113                 if (i != pos) {
8114                         memcpy(&instructions[pos], instr, sizeof(*instr));
8115                         memcpy(&instruction_data[pos], data, sizeof(*data));
8116                 }
8117
8118                 pos++;
8119         }
8120
8121         return pos;
8122 }
8123
8124 static int
8125 instr_pattern_extract_many_search(struct instruction *instr,
8126                                   struct instruction_data *data,
8127                                   uint32_t n_instr,
8128                                   uint32_t *n_pattern_instr)
8129 {
8130         uint32_t i;
8131
8132         for (i = 0; i < n_instr; i++) {
8133                 if (data[i].invalid)
8134                         break;
8135
8136                 if (instr[i].type != INSTR_HDR_EXTRACT)
8137                         break;
8138
8139                 if (i == RTE_DIM(instr->io.hdr.header_id))
8140                         break;
8141
8142                 if (i && data[i].n_users)
8143                         break;
8144         }
8145
8146         if (i < 2)
8147                 return 0;
8148
8149         *n_pattern_instr = i;
8150         return 1;
8151 }
8152
8153 static void
8154 instr_pattern_extract_many_replace(struct instruction *instr,
8155                                    struct instruction_data *data,
8156                                    uint32_t n_instr)
8157 {
8158         uint32_t i;
8159
8160         for (i = 1; i < n_instr; i++) {
8161                 instr[0].type++;
8162                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8163                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8164                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8165
8166                 data[i].invalid = 1;
8167         }
8168 }
8169
8170 static uint32_t
8171 instr_pattern_extract_many_optimize(struct instruction *instructions,
8172                                     struct instruction_data *instruction_data,
8173                                     uint32_t n_instructions)
8174 {
8175         uint32_t i;
8176
8177         for (i = 0; i < n_instructions; ) {
8178                 struct instruction *instr = &instructions[i];
8179                 struct instruction_data *data = &instruction_data[i];
8180                 uint32_t n_instr = 0;
8181                 int detected;
8182
8183                 /* Extract many. */
8184                 detected = instr_pattern_extract_many_search(instr,
8185                                                              data,
8186                                                              n_instructions - i,
8187                                                              &n_instr);
8188                 if (detected) {
8189                         instr_pattern_extract_many_replace(instr,
8190                                                            data,
8191                                                            n_instr);
8192                         i += n_instr;
8193                         continue;
8194                 }
8195
8196                 /* No pattern starting at the current instruction. */
8197                 i++;
8198         }
8199
8200         /* Eliminate the invalid instructions that have been optimized out. */
8201         n_instructions = instr_compact(instructions,
8202                                        instruction_data,
8203                                        n_instructions);
8204
8205         return n_instructions;
8206 }
8207
8208 static int
8209 instr_pattern_emit_many_tx_search(struct instruction *instr,
8210                                   struct instruction_data *data,
8211                                   uint32_t n_instr,
8212                                   uint32_t *n_pattern_instr)
8213 {
8214         uint32_t i;
8215
8216         for (i = 0; i < n_instr; i++) {
8217                 if (data[i].invalid)
8218                         break;
8219
8220                 if (instr[i].type != INSTR_HDR_EMIT)
8221                         break;
8222
8223                 if (i == RTE_DIM(instr->io.hdr.header_id))
8224                         break;
8225
8226                 if (i && data[i].n_users)
8227                         break;
8228         }
8229
8230         if (!i)
8231                 return 0;
8232
8233         if (!instruction_is_tx(instr[i].type))
8234                 return 0;
8235
8236         if (data[i].n_users)
8237                 return 0;
8238
8239         i++;
8240
8241         *n_pattern_instr = i;
8242         return 1;
8243 }
8244
8245 static void
8246 instr_pattern_emit_many_tx_replace(struct instruction *instr,
8247                                    struct instruction_data *data,
8248                                    uint32_t n_instr)
8249 {
8250         uint32_t i;
8251
8252         /* Any emit instruction in addition to the first one. */
8253         for (i = 1; i < n_instr - 1; i++) {
8254                 instr[0].type++;
8255                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8256                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8257                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8258
8259                 data[i].invalid = 1;
8260         }
8261
8262         /* The TX instruction is the last one in the pattern. */
8263         instr[0].type++;
8264         instr[0].io.io.offset = instr[i].io.io.offset;
8265         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
8266         data[i].invalid = 1;
8267 }
8268
8269 static uint32_t
8270 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
8271                                     struct instruction_data *instruction_data,
8272                                     uint32_t n_instructions)
8273 {
8274         uint32_t i;
8275
8276         for (i = 0; i < n_instructions; ) {
8277                 struct instruction *instr = &instructions[i];
8278                 struct instruction_data *data = &instruction_data[i];
8279                 uint32_t n_instr = 0;
8280                 int detected;
8281
8282                 /* Emit many + TX. */
8283                 detected = instr_pattern_emit_many_tx_search(instr,
8284                                                              data,
8285                                                              n_instructions - i,
8286                                                              &n_instr);
8287                 if (detected) {
8288                         instr_pattern_emit_many_tx_replace(instr,
8289                                                            data,
8290                                                            n_instr);
8291                         i += n_instr;
8292                         continue;
8293                 }
8294
8295                 /* No pattern starting at the current instruction. */
8296                 i++;
8297         }
8298
8299         /* Eliminate the invalid instructions that have been optimized out. */
8300         n_instructions = instr_compact(instructions,
8301                                        instruction_data,
8302                                        n_instructions);
8303
8304         return n_instructions;
8305 }
8306
8307 static int
8308 instr_pattern_dma_many_search(struct instruction *instr,
8309                               struct instruction_data *data,
8310                               uint32_t n_instr,
8311                               uint32_t *n_pattern_instr)
8312 {
8313         uint32_t i;
8314
8315         for (i = 0; i < n_instr; i++) {
8316                 if (data[i].invalid)
8317                         break;
8318
8319                 if (instr[i].type != INSTR_DMA_HT)
8320                         break;
8321
8322                 if (i == RTE_DIM(instr->dma.dst.header_id))
8323                         break;
8324
8325                 if (i && data[i].n_users)
8326                         break;
8327         }
8328
8329         if (i < 2)
8330                 return 0;
8331
8332         *n_pattern_instr = i;
8333         return 1;
8334 }
8335
8336 static void
8337 instr_pattern_dma_many_replace(struct instruction *instr,
8338                                struct instruction_data *data,
8339                                uint32_t n_instr)
8340 {
8341         uint32_t i;
8342
8343         for (i = 1; i < n_instr; i++) {
8344                 instr[0].type++;
8345                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
8346                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
8347                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
8348                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
8349
8350                 data[i].invalid = 1;
8351         }
8352 }
8353
8354 static uint32_t
8355 instr_pattern_dma_many_optimize(struct instruction *instructions,
8356                struct instruction_data *instruction_data,
8357                uint32_t n_instructions)
8358 {
8359         uint32_t i;
8360
8361         for (i = 0; i < n_instructions; ) {
8362                 struct instruction *instr = &instructions[i];
8363                 struct instruction_data *data = &instruction_data[i];
8364                 uint32_t n_instr = 0;
8365                 int detected;
8366
8367                 /* DMA many. */
8368                 detected = instr_pattern_dma_many_search(instr,
8369                                                          data,
8370                                                          n_instructions - i,
8371                                                          &n_instr);
8372                 if (detected) {
8373                         instr_pattern_dma_many_replace(instr, data, n_instr);
8374                         i += n_instr;
8375                         continue;
8376                 }
8377
8378                 /* No pattern starting at the current instruction. */
8379                 i++;
8380         }
8381
8382         /* Eliminate the invalid instructions that have been optimized out. */
8383         n_instructions = instr_compact(instructions,
8384                                        instruction_data,
8385                                        n_instructions);
8386
8387         return n_instructions;
8388 }
8389
8390 static uint32_t
8391 instr_optimize(struct instruction *instructions,
8392                struct instruction_data *instruction_data,
8393                uint32_t n_instructions)
8394 {
8395         /* Extract many. */
8396         n_instructions = instr_pattern_extract_many_optimize(instructions,
8397                                                              instruction_data,
8398                                                              n_instructions);
8399
8400         /* Emit many + TX. */
8401         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
8402                                                              instruction_data,
8403                                                              n_instructions);
8404
8405         /* DMA many. */
8406         n_instructions = instr_pattern_dma_many_optimize(instructions,
8407                                                          instruction_data,
8408                                                          n_instructions);
8409
8410         return n_instructions;
8411 }
8412
8413 static int
8414 instruction_config(struct rte_swx_pipeline *p,
8415                    struct action *a,
8416                    const char **instructions,
8417                    uint32_t n_instructions)
8418 {
8419         struct instruction *instr = NULL;
8420         struct instruction_data *data = NULL;
8421         int err = 0;
8422         uint32_t i;
8423
8424         CHECK(n_instructions, EINVAL);
8425         CHECK(instructions, EINVAL);
8426         for (i = 0; i < n_instructions; i++)
8427                 CHECK_INSTRUCTION(instructions[i], EINVAL);
8428
8429         /* Memory allocation. */
8430         instr = calloc(n_instructions, sizeof(struct instruction));
8431         if (!instr) {
8432                 err = -ENOMEM;
8433                 goto error;
8434         }
8435
8436         data = calloc(n_instructions, sizeof(struct instruction_data));
8437         if (!data) {
8438                 err = -ENOMEM;
8439                 goto error;
8440         }
8441
8442         for (i = 0; i < n_instructions; i++) {
8443                 char *string = strdup(instructions[i]);
8444                 if (!string) {
8445                         err = -ENOMEM;
8446                         goto error;
8447                 }
8448
8449                 err = instr_translate(p, a, string, &instr[i], &data[i]);
8450                 if (err) {
8451                         free(string);
8452                         goto error;
8453                 }
8454
8455                 free(string);
8456         }
8457
8458         err = instr_label_check(data, n_instructions);
8459         if (err)
8460                 goto error;
8461
8462         err = instr_verify(p, a, instr, data, n_instructions);
8463         if (err)
8464                 goto error;
8465
8466         n_instructions = instr_optimize(instr, data, n_instructions);
8467
8468         err = instr_jmp_resolve(instr, data, n_instructions);
8469         if (err)
8470                 goto error;
8471
8472         if (a) {
8473                 a->instructions = instr;
8474                 a->n_instructions = n_instructions;
8475         } else {
8476                 p->instructions = instr;
8477                 p->n_instructions = n_instructions;
8478         }
8479
8480         free(data);
8481         return 0;
8482
8483 error:
8484         free(data);
8485         free(instr);
8486         return err;
8487 }
8488
8489 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
8490
8491 static instr_exec_t instruction_table[] = {
8492         [INSTR_RX] = instr_rx_exec,
8493         [INSTR_TX] = instr_tx_exec,
8494         [INSTR_TX_I] = instr_tx_i_exec,
8495
8496         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
8497         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
8498         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
8499         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
8500         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
8501         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
8502         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
8503         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
8504
8505         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
8506         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
8507         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
8508         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
8509         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
8510         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
8511         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
8512         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
8513         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
8514
8515         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
8516         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
8517
8518         [INSTR_MOV] = instr_mov_exec,
8519         [INSTR_MOV_MH] = instr_mov_mh_exec,
8520         [INSTR_MOV_HM] = instr_mov_hm_exec,
8521         [INSTR_MOV_HH] = instr_mov_hh_exec,
8522         [INSTR_MOV_I] = instr_mov_i_exec,
8523
8524         [INSTR_DMA_HT] = instr_dma_ht_exec,
8525         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
8526         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
8527         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
8528         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
8529         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
8530         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
8531         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
8532
8533         [INSTR_ALU_ADD] = instr_alu_add_exec,
8534         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
8535         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
8536         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
8537         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
8538         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
8539
8540         [INSTR_ALU_SUB] = instr_alu_sub_exec,
8541         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
8542         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
8543         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
8544         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
8545         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
8546
8547         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
8548         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
8549         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
8550         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
8551
8552         [INSTR_ALU_AND] = instr_alu_and_exec,
8553         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
8554         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
8555         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
8556         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
8557
8558         [INSTR_ALU_OR] = instr_alu_or_exec,
8559         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
8560         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
8561         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
8562         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
8563
8564         [INSTR_ALU_XOR] = instr_alu_xor_exec,
8565         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
8566         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
8567         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
8568         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
8569
8570         [INSTR_ALU_SHL] = instr_alu_shl_exec,
8571         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
8572         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
8573         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
8574         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
8575         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
8576
8577         [INSTR_ALU_SHR] = instr_alu_shr_exec,
8578         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
8579         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
8580         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
8581         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
8582         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
8583
8584         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
8585         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
8586         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
8587
8588         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
8589         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
8590         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
8591         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
8592         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
8593         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
8594
8595         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
8596         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
8597         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
8598         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
8599         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
8600         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
8601         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
8602         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
8603         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
8604
8605         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
8606         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
8607         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
8608         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
8609         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
8610         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
8611         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
8612         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
8613         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
8614
8615         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
8616         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
8617         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
8618
8619         [INSTR_METER_HHM] = instr_meter_hhm_exec,
8620         [INSTR_METER_HHI] = instr_meter_hhi_exec,
8621         [INSTR_METER_HMM] = instr_meter_hmm_exec,
8622         [INSTR_METER_HMI] = instr_meter_hmi_exec,
8623         [INSTR_METER_MHM] = instr_meter_mhm_exec,
8624         [INSTR_METER_MHI] = instr_meter_mhi_exec,
8625         [INSTR_METER_MMM] = instr_meter_mmm_exec,
8626         [INSTR_METER_MMI] = instr_meter_mmi_exec,
8627         [INSTR_METER_IHM] = instr_meter_ihm_exec,
8628         [INSTR_METER_IHI] = instr_meter_ihi_exec,
8629         [INSTR_METER_IMM] = instr_meter_imm_exec,
8630         [INSTR_METER_IMI] = instr_meter_imi_exec,
8631
8632         [INSTR_TABLE] = instr_table_exec,
8633         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
8634         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
8635
8636         [INSTR_JMP] = instr_jmp_exec,
8637         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
8638         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
8639         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
8640         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
8641         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
8642         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
8643
8644         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
8645         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
8646         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
8647         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
8648         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
8649
8650         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
8651         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
8652         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
8653         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
8654         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
8655
8656         [INSTR_JMP_LT] = instr_jmp_lt_exec,
8657         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
8658         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
8659         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
8660         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
8661         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
8662
8663         [INSTR_JMP_GT] = instr_jmp_gt_exec,
8664         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
8665         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
8666         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
8667         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
8668         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
8669
8670         [INSTR_RETURN] = instr_return_exec,
8671 };
8672
8673 static inline void
8674 instr_exec(struct rte_swx_pipeline *p)
8675 {
8676         struct thread *t = &p->threads[p->thread_id];
8677         struct instruction *ip = t->ip;
8678         instr_exec_t instr = instruction_table[ip->type];
8679
8680         instr(p);
8681 }
8682
8683 /*
8684  * Action.
8685  */
8686 static struct action *
8687 action_find(struct rte_swx_pipeline *p, const char *name)
8688 {
8689         struct action *elem;
8690
8691         if (!name)
8692                 return NULL;
8693
8694         TAILQ_FOREACH(elem, &p->actions, node)
8695                 if (strcmp(elem->name, name) == 0)
8696                         return elem;
8697
8698         return NULL;
8699 }
8700
8701 static struct action *
8702 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8703 {
8704         struct action *action = NULL;
8705
8706         TAILQ_FOREACH(action, &p->actions, node)
8707                 if (action->id == id)
8708                         return action;
8709
8710         return NULL;
8711 }
8712
8713 static struct field *
8714 action_field_find(struct action *a, const char *name)
8715 {
8716         return a->st ? struct_type_field_find(a->st, name) : NULL;
8717 }
8718
8719 static struct field *
8720 action_field_parse(struct action *action, const char *name)
8721 {
8722         if (name[0] != 't' || name[1] != '.')
8723                 return NULL;
8724
8725         return action_field_find(action, &name[2]);
8726 }
8727
8728 int
8729 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
8730                                const char *name,
8731                                const char *args_struct_type_name,
8732                                const char **instructions,
8733                                uint32_t n_instructions)
8734 {
8735         struct struct_type *args_struct_type;
8736         struct action *a;
8737         int err;
8738
8739         CHECK(p, EINVAL);
8740
8741         CHECK_NAME(name, EINVAL);
8742         CHECK(!action_find(p, name), EEXIST);
8743
8744         if (args_struct_type_name) {
8745                 CHECK_NAME(args_struct_type_name, EINVAL);
8746                 args_struct_type = struct_type_find(p, args_struct_type_name);
8747                 CHECK(args_struct_type, EINVAL);
8748         } else {
8749                 args_struct_type = NULL;
8750         }
8751
8752         /* Node allocation. */
8753         a = calloc(1, sizeof(struct action));
8754         CHECK(a, ENOMEM);
8755
8756         /* Node initialization. */
8757         strcpy(a->name, name);
8758         a->st = args_struct_type;
8759         a->id = p->n_actions;
8760
8761         /* Instruction translation. */
8762         err = instruction_config(p, a, instructions, n_instructions);
8763         if (err) {
8764                 free(a);
8765                 return err;
8766         }
8767
8768         /* Node add to tailq. */
8769         TAILQ_INSERT_TAIL(&p->actions, a, node);
8770         p->n_actions++;
8771
8772         return 0;
8773 }
8774
8775 static int
8776 action_build(struct rte_swx_pipeline *p)
8777 {
8778         struct action *action;
8779
8780         p->action_instructions = calloc(p->n_actions,
8781                                         sizeof(struct instruction *));
8782         CHECK(p->action_instructions, ENOMEM);
8783
8784         TAILQ_FOREACH(action, &p->actions, node)
8785                 p->action_instructions[action->id] = action->instructions;
8786
8787         return 0;
8788 }
8789
8790 static void
8791 action_build_free(struct rte_swx_pipeline *p)
8792 {
8793         free(p->action_instructions);
8794         p->action_instructions = NULL;
8795 }
8796
8797 static void
8798 action_free(struct rte_swx_pipeline *p)
8799 {
8800         action_build_free(p);
8801
8802         for ( ; ; ) {
8803                 struct action *action;
8804
8805                 action = TAILQ_FIRST(&p->actions);
8806                 if (!action)
8807                         break;
8808
8809                 TAILQ_REMOVE(&p->actions, action, node);
8810                 free(action->instructions);
8811                 free(action);
8812         }
8813 }
8814
8815 /*
8816  * Table.
8817  */
8818 static struct table_type *
8819 table_type_find(struct rte_swx_pipeline *p, const char *name)
8820 {
8821         struct table_type *elem;
8822
8823         TAILQ_FOREACH(elem, &p->table_types, node)
8824                 if (strcmp(elem->name, name) == 0)
8825                         return elem;
8826
8827         return NULL;
8828 }
8829
8830 static struct table_type *
8831 table_type_resolve(struct rte_swx_pipeline *p,
8832                    const char *recommended_type_name,
8833                    enum rte_swx_table_match_type match_type)
8834 {
8835         struct table_type *elem;
8836
8837         /* Only consider the recommended type if the match type is correct. */
8838         if (recommended_type_name)
8839                 TAILQ_FOREACH(elem, &p->table_types, node)
8840                         if (!strcmp(elem->name, recommended_type_name) &&
8841                             (elem->match_type == match_type))
8842                                 return elem;
8843
8844         /* Ignore the recommended type and get the first element with this match
8845          * type.
8846          */
8847         TAILQ_FOREACH(elem, &p->table_types, node)
8848                 if (elem->match_type == match_type)
8849                         return elem;
8850
8851         return NULL;
8852 }
8853
8854 static struct table *
8855 table_find(struct rte_swx_pipeline *p, const char *name)
8856 {
8857         struct table *elem;
8858
8859         TAILQ_FOREACH(elem, &p->tables, node)
8860                 if (strcmp(elem->name, name) == 0)
8861                         return elem;
8862
8863         return NULL;
8864 }
8865
8866 static struct table *
8867 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8868 {
8869         struct table *table = NULL;
8870
8871         TAILQ_FOREACH(table, &p->tables, node)
8872                 if (table->id == id)
8873                         return table;
8874
8875         return NULL;
8876 }
8877
8878 int
8879 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
8880                                      const char *name,
8881                                      enum rte_swx_table_match_type match_type,
8882                                      struct rte_swx_table_ops *ops)
8883 {
8884         struct table_type *elem;
8885
8886         CHECK(p, EINVAL);
8887
8888         CHECK_NAME(name, EINVAL);
8889         CHECK(!table_type_find(p, name), EEXIST);
8890
8891         CHECK(ops, EINVAL);
8892         CHECK(ops->create, EINVAL);
8893         CHECK(ops->lkp, EINVAL);
8894         CHECK(ops->free, EINVAL);
8895
8896         /* Node allocation. */
8897         elem = calloc(1, sizeof(struct table_type));
8898         CHECK(elem, ENOMEM);
8899
8900         /* Node initialization. */
8901         strcpy(elem->name, name);
8902         elem->match_type = match_type;
8903         memcpy(&elem->ops, ops, sizeof(*ops));
8904
8905         /* Node add to tailq. */
8906         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
8907
8908         return 0;
8909 }
8910
8911 static enum rte_swx_table_match_type
8912 table_match_type_resolve(struct rte_swx_match_field_params *fields,
8913                          uint32_t n_fields,
8914                          uint32_t max_offset_field_id)
8915 {
8916         uint32_t n_fields_em = 0, i;
8917
8918         for (i = 0; i < n_fields; i++)
8919                 if (fields[i].match_type == RTE_SWX_TABLE_MATCH_EXACT)
8920                         n_fields_em++;
8921
8922         if (n_fields_em == n_fields)
8923                 return RTE_SWX_TABLE_MATCH_EXACT;
8924
8925         if ((n_fields_em == n_fields - 1) &&
8926             (fields[max_offset_field_id].match_type == RTE_SWX_TABLE_MATCH_LPM))
8927                 return RTE_SWX_TABLE_MATCH_LPM;
8928
8929         return RTE_SWX_TABLE_MATCH_WILDCARD;
8930 }
8931
8932 static int
8933 table_match_fields_check(struct rte_swx_pipeline *p,
8934                          struct rte_swx_pipeline_table_params *params,
8935                          struct header **header,
8936                          uint32_t *min_offset_field_id,
8937                          uint32_t *max_offset_field_id)
8938 {
8939         struct header *h0 = NULL;
8940         struct field *hf, *mf;
8941         uint32_t *offset = NULL, min_offset, max_offset, min_offset_pos, max_offset_pos, i;
8942         int status = 0;
8943
8944         /* Return if no match fields. */
8945         if (!params->n_fields) {
8946                 if (params->fields) {
8947                         status = -EINVAL;
8948                         goto end;
8949                 }
8950
8951                 return 0;
8952         }
8953
8954         /* Memory allocation. */
8955         offset = calloc(params->n_fields, sizeof(uint32_t));
8956         if (!offset) {
8957                 status = -ENOMEM;
8958                 goto end;
8959         }
8960
8961         /* Check that all the match fields belong to either the same header or
8962          * to the meta-data.
8963          */
8964         hf = header_field_parse(p, params->fields[0].name, &h0);
8965         mf = metadata_field_parse(p, params->fields[0].name);
8966         if (!hf && !mf) {
8967                 status = -EINVAL;
8968                 goto end;
8969         }
8970
8971         offset[0] = h0 ? hf->offset : mf->offset;
8972
8973         for (i = 1; i < params->n_fields; i++)
8974                 if (h0) {
8975                         struct header *h;
8976
8977                         hf = header_field_parse(p, params->fields[i].name, &h);
8978                         if (!hf || (h->id != h0->id)) {
8979                                 status = -EINVAL;
8980                                 goto end;
8981                         }
8982
8983                         offset[i] = hf->offset;
8984                 } else {
8985                         mf = metadata_field_parse(p, params->fields[i].name);
8986                         if (!mf) {
8987                                 status = -EINVAL;
8988                                 goto end;
8989                         }
8990
8991                         offset[i] = mf->offset;
8992                 }
8993
8994         /* Check that there are no duplicated match fields. */
8995         for (i = 0; i < params->n_fields; i++) {
8996                 uint32_t j;
8997
8998                 for (j = 0; j < i; j++)
8999                         if (offset[j] == offset[i]) {
9000                                 status = -EINVAL;
9001                                 goto end;
9002                         }
9003         }
9004
9005         /* Find the min and max offset fields. */
9006         min_offset = offset[0];
9007         max_offset = offset[0];
9008         min_offset_pos = 0;
9009         max_offset_pos = 0;
9010
9011         for (i = 1; i < params->n_fields; i++) {
9012                 if (offset[i] < min_offset) {
9013                         min_offset = offset[i];
9014                         min_offset_pos = i;
9015                 }
9016
9017                 if (offset[i] > max_offset) {
9018                         max_offset = offset[i];
9019                         max_offset_pos = i;
9020                 }
9021         }
9022
9023         /* Return. */
9024         if (header)
9025                 *header = h0;
9026
9027         if (min_offset_field_id)
9028                 *min_offset_field_id = min_offset_pos;
9029
9030         if (max_offset_field_id)
9031                 *max_offset_field_id = max_offset_pos;
9032
9033 end:
9034         free(offset);
9035         return status;
9036 }
9037
9038 int
9039 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
9040                               const char *name,
9041                               struct rte_swx_pipeline_table_params *params,
9042                               const char *recommended_table_type_name,
9043                               const char *args,
9044                               uint32_t size)
9045 {
9046         struct table_type *type;
9047         struct table *t;
9048         struct action *default_action;
9049         struct header *header = NULL;
9050         uint32_t action_data_size_max = 0, min_offset_field_id = 0, max_offset_field_id = 0, i;
9051         int status = 0;
9052
9053         CHECK(p, EINVAL);
9054
9055         CHECK_NAME(name, EINVAL);
9056         CHECK(!table_find(p, name), EEXIST);
9057
9058         CHECK(params, EINVAL);
9059
9060         /* Match checks. */
9061         status = table_match_fields_check(p,
9062                                           params,
9063                                           &header,
9064                                           &min_offset_field_id,
9065                                           &max_offset_field_id);
9066         if (status)
9067                 return status;
9068
9069         /* Action checks. */
9070         CHECK(params->n_actions, EINVAL);
9071         CHECK(params->action_names, EINVAL);
9072         for (i = 0; i < params->n_actions; i++) {
9073                 const char *action_name = params->action_names[i];
9074                 struct action *a;
9075                 uint32_t action_data_size;
9076
9077                 CHECK_NAME(action_name, EINVAL);
9078
9079                 a = action_find(p, action_name);
9080                 CHECK(a, EINVAL);
9081
9082                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
9083                 if (action_data_size > action_data_size_max)
9084                         action_data_size_max = action_data_size;
9085         }
9086
9087         CHECK_NAME(params->default_action_name, EINVAL);
9088         for (i = 0; i < p->n_actions; i++)
9089                 if (!strcmp(params->action_names[i],
9090                             params->default_action_name))
9091                         break;
9092         CHECK(i < params->n_actions, EINVAL);
9093         default_action = action_find(p, params->default_action_name);
9094         CHECK((default_action->st && params->default_action_data) ||
9095               !params->default_action_data, EINVAL);
9096
9097         /* Table type checks. */
9098         if (recommended_table_type_name)
9099                 CHECK_NAME(recommended_table_type_name, EINVAL);
9100
9101         if (params->n_fields) {
9102                 enum rte_swx_table_match_type match_type;
9103
9104                 match_type = table_match_type_resolve(params->fields,
9105                                                       params->n_fields,
9106                                                       max_offset_field_id);
9107                 type = table_type_resolve(p,
9108                                           recommended_table_type_name,
9109                                           match_type);
9110                 CHECK(type, EINVAL);
9111         } else {
9112                 type = NULL;
9113         }
9114
9115         /* Memory allocation. */
9116         t = calloc(1, sizeof(struct table));
9117         CHECK(t, ENOMEM);
9118
9119         t->fields = calloc(params->n_fields, sizeof(struct match_field));
9120         if (!t->fields) {
9121                 free(t);
9122                 CHECK(0, ENOMEM);
9123         }
9124
9125         t->actions = calloc(params->n_actions, sizeof(struct action *));
9126         if (!t->actions) {
9127                 free(t->fields);
9128                 free(t);
9129                 CHECK(0, ENOMEM);
9130         }
9131
9132         if (action_data_size_max) {
9133                 t->default_action_data = calloc(1, action_data_size_max);
9134                 if (!t->default_action_data) {
9135                         free(t->actions);
9136                         free(t->fields);
9137                         free(t);
9138                         CHECK(0, ENOMEM);
9139                 }
9140         }
9141
9142         /* Node initialization. */
9143         strcpy(t->name, name);
9144         if (args && args[0])
9145                 strcpy(t->args, args);
9146         t->type = type;
9147
9148         for (i = 0; i < params->n_fields; i++) {
9149                 struct rte_swx_match_field_params *field = &params->fields[i];
9150                 struct match_field *f = &t->fields[i];
9151
9152                 f->match_type = field->match_type;
9153                 f->field = header ?
9154                         header_field_parse(p, field->name, NULL) :
9155                         metadata_field_parse(p, field->name);
9156         }
9157         t->n_fields = params->n_fields;
9158         t->header = header;
9159
9160         for (i = 0; i < params->n_actions; i++)
9161                 t->actions[i] = action_find(p, params->action_names[i]);
9162         t->default_action = default_action;
9163         if (default_action->st)
9164                 memcpy(t->default_action_data,
9165                        params->default_action_data,
9166                        default_action->st->n_bits / 8);
9167         t->n_actions = params->n_actions;
9168         t->default_action_is_const = params->default_action_is_const;
9169         t->action_data_size_max = action_data_size_max;
9170
9171         t->size = size;
9172         t->id = p->n_tables;
9173
9174         /* Node add to tailq. */
9175         TAILQ_INSERT_TAIL(&p->tables, t, node);
9176         p->n_tables++;
9177
9178         return 0;
9179 }
9180
9181 static struct rte_swx_table_params *
9182 table_params_get(struct table *table)
9183 {
9184         struct rte_swx_table_params *params;
9185         struct field *first, *last;
9186         uint8_t *key_mask;
9187         uint32_t key_size, key_offset, action_data_size, i;
9188
9189         /* Memory allocation. */
9190         params = calloc(1, sizeof(struct rte_swx_table_params));
9191         if (!params)
9192                 return NULL;
9193
9194         /* Find first (smallest offset) and last (biggest offset) match fields. */
9195         first = table->fields[0].field;
9196         last = table->fields[0].field;
9197
9198         for (i = 0; i < table->n_fields; i++) {
9199                 struct field *f = table->fields[i].field;
9200
9201                 if (f->offset < first->offset)
9202                         first = f;
9203
9204                 if (f->offset > last->offset)
9205                         last = f;
9206         }
9207
9208         /* Key offset and size. */
9209         key_offset = first->offset / 8;
9210         key_size = (last->offset + last->n_bits - first->offset) / 8;
9211
9212         /* Memory allocation. */
9213         key_mask = calloc(1, key_size);
9214         if (!key_mask) {
9215                 free(params);
9216                 return NULL;
9217         }
9218
9219         /* Key mask. */
9220         for (i = 0; i < table->n_fields; i++) {
9221                 struct field *f = table->fields[i].field;
9222                 uint32_t start = (f->offset - first->offset) / 8;
9223                 size_t size = f->n_bits / 8;
9224
9225                 memset(&key_mask[start], 0xFF, size);
9226         }
9227
9228         /* Action data size. */
9229         action_data_size = 0;
9230         for (i = 0; i < table->n_actions; i++) {
9231                 struct action *action = table->actions[i];
9232                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
9233
9234                 if (ads > action_data_size)
9235                         action_data_size = ads;
9236         }
9237
9238         /* Fill in. */
9239         params->match_type = table->type->match_type;
9240         params->key_size = key_size;
9241         params->key_offset = key_offset;
9242         params->key_mask0 = key_mask;
9243         params->action_data_size = action_data_size;
9244         params->n_keys_max = table->size;
9245
9246         return params;
9247 }
9248
9249 static void
9250 table_params_free(struct rte_swx_table_params *params)
9251 {
9252         if (!params)
9253                 return;
9254
9255         free(params->key_mask0);
9256         free(params);
9257 }
9258
9259 static int
9260 table_state_build(struct rte_swx_pipeline *p)
9261 {
9262         struct table *table;
9263
9264         p->table_state = calloc(p->n_tables,
9265                                 sizeof(struct rte_swx_table_state));
9266         CHECK(p->table_state, ENOMEM);
9267
9268         TAILQ_FOREACH(table, &p->tables, node) {
9269                 struct rte_swx_table_state *ts = &p->table_state[table->id];
9270
9271                 if (table->type) {
9272                         struct rte_swx_table_params *params;
9273
9274                         /* ts->obj. */
9275                         params = table_params_get(table);
9276                         CHECK(params, ENOMEM);
9277
9278                         ts->obj = table->type->ops.create(params,
9279                                 NULL,
9280                                 table->args,
9281                                 p->numa_node);
9282
9283                         table_params_free(params);
9284                         CHECK(ts->obj, ENODEV);
9285                 }
9286
9287                 /* ts->default_action_data. */
9288                 if (table->action_data_size_max) {
9289                         ts->default_action_data =
9290                                 malloc(table->action_data_size_max);
9291                         CHECK(ts->default_action_data, ENOMEM);
9292
9293                         memcpy(ts->default_action_data,
9294                                table->default_action_data,
9295                                table->action_data_size_max);
9296                 }
9297
9298                 /* ts->default_action_id. */
9299                 ts->default_action_id = table->default_action->id;
9300         }
9301
9302         return 0;
9303 }
9304
9305 static void
9306 table_state_build_free(struct rte_swx_pipeline *p)
9307 {
9308         uint32_t i;
9309
9310         if (!p->table_state)
9311                 return;
9312
9313         for (i = 0; i < p->n_tables; i++) {
9314                 struct rte_swx_table_state *ts = &p->table_state[i];
9315                 struct table *table = table_find_by_id(p, i);
9316
9317                 /* ts->obj. */
9318                 if (table->type && ts->obj)
9319                         table->type->ops.free(ts->obj);
9320
9321                 /* ts->default_action_data. */
9322                 free(ts->default_action_data);
9323         }
9324
9325         free(p->table_state);
9326         p->table_state = NULL;
9327 }
9328
9329 static void
9330 table_state_free(struct rte_swx_pipeline *p)
9331 {
9332         table_state_build_free(p);
9333 }
9334
9335 static int
9336 table_stub_lkp(void *table __rte_unused,
9337                void *mailbox __rte_unused,
9338                uint8_t **key __rte_unused,
9339                uint64_t *action_id __rte_unused,
9340                uint8_t **action_data __rte_unused,
9341                int *hit)
9342 {
9343         *hit = 0;
9344         return 1; /* DONE. */
9345 }
9346
9347 static int
9348 table_build(struct rte_swx_pipeline *p)
9349 {
9350         uint32_t i;
9351
9352         /* Per pipeline: table statistics. */
9353         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
9354         CHECK(p->table_stats, ENOMEM);
9355
9356         for (i = 0; i < p->n_tables; i++) {
9357                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9358                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
9359         }
9360
9361         /* Per thread: table runt-time. */
9362         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9363                 struct thread *t = &p->threads[i];
9364                 struct table *table;
9365
9366                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
9367                 CHECK(t->tables, ENOMEM);
9368
9369                 TAILQ_FOREACH(table, &p->tables, node) {
9370                         struct table_runtime *r = &t->tables[table->id];
9371
9372                         if (table->type) {
9373                                 uint64_t size;
9374
9375                                 size = table->type->ops.mailbox_size_get();
9376
9377                                 /* r->func. */
9378                                 r->func = table->type->ops.lkp;
9379
9380                                 /* r->mailbox. */
9381                                 if (size) {
9382                                         r->mailbox = calloc(1, size);
9383                                         CHECK(r->mailbox, ENOMEM);
9384                                 }
9385
9386                                 /* r->key. */
9387                                 r->key = table->header ?
9388                                         &t->structs[table->header->struct_id] :
9389                                         &t->structs[p->metadata_struct_id];
9390                         } else {
9391                                 r->func = table_stub_lkp;
9392                         }
9393                 }
9394         }
9395
9396         return 0;
9397 }
9398
9399 static void
9400 table_build_free(struct rte_swx_pipeline *p)
9401 {
9402         uint32_t i;
9403
9404         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9405                 struct thread *t = &p->threads[i];
9406                 uint32_t j;
9407
9408                 if (!t->tables)
9409                         continue;
9410
9411                 for (j = 0; j < p->n_tables; j++) {
9412                         struct table_runtime *r = &t->tables[j];
9413
9414                         free(r->mailbox);
9415                 }
9416
9417                 free(t->tables);
9418                 t->tables = NULL;
9419         }
9420
9421         if (p->table_stats) {
9422                 for (i = 0; i < p->n_tables; i++)
9423                         free(p->table_stats[i].n_pkts_action);
9424
9425                 free(p->table_stats);
9426         }
9427 }
9428
9429 static void
9430 table_free(struct rte_swx_pipeline *p)
9431 {
9432         table_build_free(p);
9433
9434         /* Tables. */
9435         for ( ; ; ) {
9436                 struct table *elem;
9437
9438                 elem = TAILQ_FIRST(&p->tables);
9439                 if (!elem)
9440                         break;
9441
9442                 TAILQ_REMOVE(&p->tables, elem, node);
9443                 free(elem->fields);
9444                 free(elem->actions);
9445                 free(elem->default_action_data);
9446                 free(elem);
9447         }
9448
9449         /* Table types. */
9450         for ( ; ; ) {
9451                 struct table_type *elem;
9452
9453                 elem = TAILQ_FIRST(&p->table_types);
9454                 if (!elem)
9455                         break;
9456
9457                 TAILQ_REMOVE(&p->table_types, elem, node);
9458                 free(elem);
9459         }
9460 }
9461
9462 /*
9463  * Register array.
9464  */
9465 static struct regarray *
9466 regarray_find(struct rte_swx_pipeline *p, const char *name)
9467 {
9468         struct regarray *elem;
9469
9470         TAILQ_FOREACH(elem, &p->regarrays, node)
9471                 if (!strcmp(elem->name, name))
9472                         return elem;
9473
9474         return NULL;
9475 }
9476
9477 static struct regarray *
9478 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9479 {
9480         struct regarray *elem = NULL;
9481
9482         TAILQ_FOREACH(elem, &p->regarrays, node)
9483                 if (elem->id == id)
9484                         return elem;
9485
9486         return NULL;
9487 }
9488
9489 int
9490 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9491                               const char *name,
9492                               uint32_t size,
9493                               uint64_t init_val)
9494 {
9495         struct regarray *r;
9496
9497         CHECK(p, EINVAL);
9498
9499         CHECK_NAME(name, EINVAL);
9500         CHECK(!regarray_find(p, name), EEXIST);
9501
9502         CHECK(size, EINVAL);
9503         size = rte_align32pow2(size);
9504
9505         /* Memory allocation. */
9506         r = calloc(1, sizeof(struct regarray));
9507         CHECK(r, ENOMEM);
9508
9509         /* Node initialization. */
9510         strcpy(r->name, name);
9511         r->init_val = init_val;
9512         r->size = size;
9513         r->id = p->n_regarrays;
9514
9515         /* Node add to tailq. */
9516         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9517         p->n_regarrays++;
9518
9519         return 0;
9520 }
9521
9522 static int
9523 regarray_build(struct rte_swx_pipeline *p)
9524 {
9525         struct regarray *regarray;
9526
9527         if (!p->n_regarrays)
9528                 return 0;
9529
9530         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9531         CHECK(p->regarray_runtime, ENOMEM);
9532
9533         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9534                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9535                 uint32_t i;
9536
9537                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9538                                          RTE_CACHE_LINE_SIZE,
9539                                          p->numa_node);
9540                 CHECK(r->regarray, ENOMEM);
9541
9542                 if (regarray->init_val)
9543                         for (i = 0; i < regarray->size; i++)
9544                                 r->regarray[i] = regarray->init_val;
9545
9546                 r->size_mask = regarray->size - 1;
9547         }
9548
9549         return 0;
9550 }
9551
9552 static void
9553 regarray_build_free(struct rte_swx_pipeline *p)
9554 {
9555         uint32_t i;
9556
9557         if (!p->regarray_runtime)
9558                 return;
9559
9560         for (i = 0; i < p->n_regarrays; i++) {
9561                 struct regarray *regarray = regarray_find_by_id(p, i);
9562                 struct regarray_runtime *r = &p->regarray_runtime[i];
9563
9564                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9565         }
9566
9567         free(p->regarray_runtime);
9568         p->regarray_runtime = NULL;
9569 }
9570
9571 static void
9572 regarray_free(struct rte_swx_pipeline *p)
9573 {
9574         regarray_build_free(p);
9575
9576         for ( ; ; ) {
9577                 struct regarray *elem;
9578
9579                 elem = TAILQ_FIRST(&p->regarrays);
9580                 if (!elem)
9581                         break;
9582
9583                 TAILQ_REMOVE(&p->regarrays, elem, node);
9584                 free(elem);
9585         }
9586 }
9587
9588 /*
9589  * Meter array.
9590  */
9591 static struct meter_profile *
9592 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9593 {
9594         struct meter_profile *elem;
9595
9596         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9597                 if (!strcmp(elem->name, name))
9598                         return elem;
9599
9600         return NULL;
9601 }
9602
9603 static struct metarray *
9604 metarray_find(struct rte_swx_pipeline *p, const char *name)
9605 {
9606         struct metarray *elem;
9607
9608         TAILQ_FOREACH(elem, &p->metarrays, node)
9609                 if (!strcmp(elem->name, name))
9610                         return elem;
9611
9612         return NULL;
9613 }
9614
9615 static struct metarray *
9616 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9617 {
9618         struct metarray *elem = NULL;
9619
9620         TAILQ_FOREACH(elem, &p->metarrays, node)
9621                 if (elem->id == id)
9622                         return elem;
9623
9624         return NULL;
9625 }
9626
9627 int
9628 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9629                                  const char *name,
9630                                  uint32_t size)
9631 {
9632         struct metarray *m;
9633
9634         CHECK(p, EINVAL);
9635
9636         CHECK_NAME(name, EINVAL);
9637         CHECK(!metarray_find(p, name), EEXIST);
9638
9639         CHECK(size, EINVAL);
9640         size = rte_align32pow2(size);
9641
9642         /* Memory allocation. */
9643         m = calloc(1, sizeof(struct metarray));
9644         CHECK(m, ENOMEM);
9645
9646         /* Node initialization. */
9647         strcpy(m->name, name);
9648         m->size = size;
9649         m->id = p->n_metarrays;
9650
9651         /* Node add to tailq. */
9652         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9653         p->n_metarrays++;
9654
9655         return 0;
9656 }
9657
9658 struct meter_profile meter_profile_default = {
9659         .node = {0},
9660         .name = "",
9661         .params = {0},
9662
9663         .profile = {
9664                 .cbs = 10000,
9665                 .pbs = 10000,
9666                 .cir_period = 1,
9667                 .cir_bytes_per_period = 1,
9668                 .pir_period = 1,
9669                 .pir_bytes_per_period = 1,
9670         },
9671
9672         .n_users = 0,
9673 };
9674
9675 static void
9676 meter_init(struct meter *m)
9677 {
9678         memset(m, 0, sizeof(struct meter));
9679         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9680         m->profile = &meter_profile_default;
9681         m->color_mask = RTE_COLOR_GREEN;
9682
9683         meter_profile_default.n_users++;
9684 }
9685
9686 static int
9687 metarray_build(struct rte_swx_pipeline *p)
9688 {
9689         struct metarray *m;
9690
9691         if (!p->n_metarrays)
9692                 return 0;
9693
9694         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9695         CHECK(p->metarray_runtime, ENOMEM);
9696
9697         TAILQ_FOREACH(m, &p->metarrays, node) {
9698                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9699                 uint32_t i;
9700
9701                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9702                                          RTE_CACHE_LINE_SIZE,
9703                                          p->numa_node);
9704                 CHECK(r->metarray, ENOMEM);
9705
9706                 for (i = 0; i < m->size; i++)
9707                         meter_init(&r->metarray[i]);
9708
9709                 r->size_mask = m->size - 1;
9710         }
9711
9712         return 0;
9713 }
9714
9715 static void
9716 metarray_build_free(struct rte_swx_pipeline *p)
9717 {
9718         uint32_t i;
9719
9720         if (!p->metarray_runtime)
9721                 return;
9722
9723         for (i = 0; i < p->n_metarrays; i++) {
9724                 struct metarray *m = metarray_find_by_id(p, i);
9725                 struct metarray_runtime *r = &p->metarray_runtime[i];
9726
9727                 env_free(r->metarray, m->size * sizeof(struct meter));
9728         }
9729
9730         free(p->metarray_runtime);
9731         p->metarray_runtime = NULL;
9732 }
9733
9734 static void
9735 metarray_free(struct rte_swx_pipeline *p)
9736 {
9737         metarray_build_free(p);
9738
9739         /* Meter arrays. */
9740         for ( ; ; ) {
9741                 struct metarray *elem;
9742
9743                 elem = TAILQ_FIRST(&p->metarrays);
9744                 if (!elem)
9745                         break;
9746
9747                 TAILQ_REMOVE(&p->metarrays, elem, node);
9748                 free(elem);
9749         }
9750
9751         /* Meter profiles. */
9752         for ( ; ; ) {
9753                 struct meter_profile *elem;
9754
9755                 elem = TAILQ_FIRST(&p->meter_profiles);
9756                 if (!elem)
9757                         break;
9758
9759                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9760                 free(elem);
9761         }
9762 }
9763
9764 /*
9765  * Pipeline.
9766  */
9767 int
9768 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9769 {
9770         struct rte_swx_pipeline *pipeline;
9771
9772         /* Check input parameters. */
9773         CHECK(p, EINVAL);
9774
9775         /* Memory allocation. */
9776         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9777         CHECK(pipeline, ENOMEM);
9778
9779         /* Initialization. */
9780         TAILQ_INIT(&pipeline->struct_types);
9781         TAILQ_INIT(&pipeline->port_in_types);
9782         TAILQ_INIT(&pipeline->ports_in);
9783         TAILQ_INIT(&pipeline->port_out_types);
9784         TAILQ_INIT(&pipeline->ports_out);
9785         TAILQ_INIT(&pipeline->extern_types);
9786         TAILQ_INIT(&pipeline->extern_objs);
9787         TAILQ_INIT(&pipeline->extern_funcs);
9788         TAILQ_INIT(&pipeline->headers);
9789         TAILQ_INIT(&pipeline->actions);
9790         TAILQ_INIT(&pipeline->table_types);
9791         TAILQ_INIT(&pipeline->tables);
9792         TAILQ_INIT(&pipeline->regarrays);
9793         TAILQ_INIT(&pipeline->meter_profiles);
9794         TAILQ_INIT(&pipeline->metarrays);
9795
9796         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9797         pipeline->numa_node = numa_node;
9798
9799         *p = pipeline;
9800         return 0;
9801 }
9802
9803 void
9804 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9805 {
9806         if (!p)
9807                 return;
9808
9809         free(p->instructions);
9810
9811         metarray_free(p);
9812         regarray_free(p);
9813         table_state_free(p);
9814         table_free(p);
9815         action_free(p);
9816         metadata_free(p);
9817         header_free(p);
9818         extern_func_free(p);
9819         extern_obj_free(p);
9820         port_out_free(p);
9821         port_in_free(p);
9822         struct_free(p);
9823
9824         free(p);
9825 }
9826
9827 int
9828 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9829                                      const char **instructions,
9830                                      uint32_t n_instructions)
9831 {
9832         int err;
9833         uint32_t i;
9834
9835         err = instruction_config(p, NULL, instructions, n_instructions);
9836         if (err)
9837                 return err;
9838
9839         /* Thread instruction pointer reset. */
9840         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9841                 struct thread *t = &p->threads[i];
9842
9843                 thread_ip_reset(p, t);
9844         }
9845
9846         return 0;
9847 }
9848
9849 int
9850 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9851 {
9852         int status;
9853
9854         CHECK(p, EINVAL);
9855         CHECK(p->build_done == 0, EEXIST);
9856
9857         status = port_in_build(p);
9858         if (status)
9859                 goto error;
9860
9861         status = port_out_build(p);
9862         if (status)
9863                 goto error;
9864
9865         status = struct_build(p);
9866         if (status)
9867                 goto error;
9868
9869         status = extern_obj_build(p);
9870         if (status)
9871                 goto error;
9872
9873         status = extern_func_build(p);
9874         if (status)
9875                 goto error;
9876
9877         status = header_build(p);
9878         if (status)
9879                 goto error;
9880
9881         status = metadata_build(p);
9882         if (status)
9883                 goto error;
9884
9885         status = action_build(p);
9886         if (status)
9887                 goto error;
9888
9889         status = table_build(p);
9890         if (status)
9891                 goto error;
9892
9893         status = table_state_build(p);
9894         if (status)
9895                 goto error;
9896
9897         status = regarray_build(p);
9898         if (status)
9899                 goto error;
9900
9901         status = metarray_build(p);
9902         if (status)
9903                 goto error;
9904
9905         p->build_done = 1;
9906         return 0;
9907
9908 error:
9909         metarray_build_free(p);
9910         regarray_build_free(p);
9911         table_state_build_free(p);
9912         table_build_free(p);
9913         action_build_free(p);
9914         metadata_build_free(p);
9915         header_build_free(p);
9916         extern_func_build_free(p);
9917         extern_obj_build_free(p);
9918         port_out_build_free(p);
9919         port_in_build_free(p);
9920         struct_build_free(p);
9921
9922         return status;
9923 }
9924
9925 void
9926 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9927 {
9928         uint32_t i;
9929
9930         for (i = 0; i < n_instructions; i++)
9931                 instr_exec(p);
9932 }
9933
9934 void
9935 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9936 {
9937         uint32_t i;
9938
9939         for (i = 0; i < p->n_ports_out; i++) {
9940                 struct port_out_runtime *port = &p->out[i];
9941
9942                 if (port->flush)
9943                         port->flush(port->obj);
9944         }
9945 }
9946
9947 /*
9948  * Control.
9949  */
9950 int
9951 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9952                               struct rte_swx_ctl_pipeline_info *pipeline)
9953 {
9954         struct action *action;
9955         struct table *table;
9956         uint32_t n_actions = 0, n_tables = 0;
9957
9958         if (!p || !pipeline)
9959                 return -EINVAL;
9960
9961         TAILQ_FOREACH(action, &p->actions, node)
9962                 n_actions++;
9963
9964         TAILQ_FOREACH(table, &p->tables, node)
9965                 n_tables++;
9966
9967         pipeline->n_ports_in = p->n_ports_in;
9968         pipeline->n_ports_out = p->n_ports_out;
9969         pipeline->n_actions = n_actions;
9970         pipeline->n_tables = n_tables;
9971         pipeline->n_regarrays = p->n_regarrays;
9972         pipeline->n_metarrays = p->n_metarrays;
9973
9974         return 0;
9975 }
9976
9977 int
9978 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9979 {
9980         if (!p || !numa_node)
9981                 return -EINVAL;
9982
9983         *numa_node = p->numa_node;
9984         return 0;
9985 }
9986
9987 int
9988 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9989                             uint32_t action_id,
9990                             struct rte_swx_ctl_action_info *action)
9991 {
9992         struct action *a = NULL;
9993
9994         if (!p || (action_id >= p->n_actions) || !action)
9995                 return -EINVAL;
9996
9997         a = action_find_by_id(p, action_id);
9998         if (!a)
9999                 return -EINVAL;
10000
10001         strcpy(action->name, a->name);
10002         action->n_args = a->st ? a->st->n_fields : 0;
10003         return 0;
10004 }
10005
10006 int
10007 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10008                                 uint32_t action_id,
10009                                 uint32_t action_arg_id,
10010                                 struct rte_swx_ctl_action_arg_info *action_arg)
10011 {
10012         struct action *a = NULL;
10013         struct field *arg = NULL;
10014
10015         if (!p || (action_id >= p->n_actions) || !action_arg)
10016                 return -EINVAL;
10017
10018         a = action_find_by_id(p, action_id);
10019         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10020                 return -EINVAL;
10021
10022         arg = &a->st->fields[action_arg_id];
10023         strcpy(action_arg->name, arg->name);
10024         action_arg->n_bits = arg->n_bits;
10025
10026         return 0;
10027 }
10028
10029 int
10030 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10031                            uint32_t table_id,
10032                            struct rte_swx_ctl_table_info *table)
10033 {
10034         struct table *t = NULL;
10035
10036         if (!p || !table)
10037                 return -EINVAL;
10038
10039         t = table_find_by_id(p, table_id);
10040         if (!t)
10041                 return -EINVAL;
10042
10043         strcpy(table->name, t->name);
10044         strcpy(table->args, t->args);
10045         table->n_match_fields = t->n_fields;
10046         table->n_actions = t->n_actions;
10047         table->default_action_is_const = t->default_action_is_const;
10048         table->size = t->size;
10049         return 0;
10050 }
10051
10052 int
10053 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10054         uint32_t table_id,
10055         uint32_t match_field_id,
10056         struct rte_swx_ctl_table_match_field_info *match_field)
10057 {
10058         struct table *t;
10059         struct match_field *f;
10060
10061         if (!p || (table_id >= p->n_tables) || !match_field)
10062                 return -EINVAL;
10063
10064         t = table_find_by_id(p, table_id);
10065         if (!t || (match_field_id >= t->n_fields))
10066                 return -EINVAL;
10067
10068         f = &t->fields[match_field_id];
10069         match_field->match_type = f->match_type;
10070         match_field->is_header = t->header ? 1 : 0;
10071         match_field->n_bits = f->field->n_bits;
10072         match_field->offset = f->field->offset;
10073
10074         return 0;
10075 }
10076
10077 int
10078 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10079         uint32_t table_id,
10080         uint32_t table_action_id,
10081         struct rte_swx_ctl_table_action_info *table_action)
10082 {
10083         struct table *t;
10084
10085         if (!p || (table_id >= p->n_tables) || !table_action)
10086                 return -EINVAL;
10087
10088         t = table_find_by_id(p, table_id);
10089         if (!t || (table_action_id >= t->n_actions))
10090                 return -EINVAL;
10091
10092         table_action->action_id = t->actions[table_action_id]->id;
10093
10094         return 0;
10095 }
10096
10097 int
10098 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10099                           uint32_t table_id,
10100                           struct rte_swx_table_ops *table_ops,
10101                           int *is_stub)
10102 {
10103         struct table *t;
10104
10105         if (!p || (table_id >= p->n_tables))
10106                 return -EINVAL;
10107
10108         t = table_find_by_id(p, table_id);
10109         if (!t)
10110                 return -EINVAL;
10111
10112         if (t->type) {
10113                 if (table_ops)
10114                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10115                 *is_stub = 0;
10116         } else {
10117                 *is_stub = 1;
10118         }
10119
10120         return 0;
10121 }
10122
10123 int
10124 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10125                                  struct rte_swx_table_state **table_state)
10126 {
10127         if (!p || !table_state || !p->build_done)
10128                 return -EINVAL;
10129
10130         *table_state = p->table_state;
10131         return 0;
10132 }
10133
10134 int
10135 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10136                                  struct rte_swx_table_state *table_state)
10137 {
10138         if (!p || !table_state || !p->build_done)
10139                 return -EINVAL;
10140
10141         p->table_state = table_state;
10142         return 0;
10143 }
10144
10145 int
10146 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10147                                         uint32_t port_id,
10148                                         struct rte_swx_port_in_stats *stats)
10149 {
10150         struct port_in *port;
10151
10152         if (!p || !stats)
10153                 return -EINVAL;
10154
10155         port = port_in_find(p, port_id);
10156         if (!port)
10157                 return -EINVAL;
10158
10159         port->type->ops.stats_read(port->obj, stats);
10160         return 0;
10161 }
10162
10163 int
10164 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10165                                          uint32_t port_id,
10166                                          struct rte_swx_port_out_stats *stats)
10167 {
10168         struct port_out *port;
10169
10170         if (!p || !stats)
10171                 return -EINVAL;
10172
10173         port = port_out_find(p, port_id);
10174         if (!port)
10175                 return -EINVAL;
10176
10177         port->type->ops.stats_read(port->obj, stats);
10178         return 0;
10179 }
10180
10181 int
10182 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10183                                       const char *table_name,
10184                                       struct rte_swx_table_stats *stats)
10185 {
10186         struct table *table;
10187         struct table_statistics *table_stats;
10188
10189         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10190                 return -EINVAL;
10191
10192         table = table_find(p, table_name);
10193         if (!table)
10194                 return -EINVAL;
10195
10196         table_stats = &p->table_stats[table->id];
10197
10198         memcpy(&stats->n_pkts_action,
10199                &table_stats->n_pkts_action,
10200                p->n_actions * sizeof(uint64_t));
10201
10202         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10203         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10204
10205         return 0;
10206 }
10207
10208 int
10209 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10210                               uint32_t regarray_id,
10211                               struct rte_swx_ctl_regarray_info *regarray)
10212 {
10213         struct regarray *r;
10214
10215         if (!p || !regarray)
10216                 return -EINVAL;
10217
10218         r = regarray_find_by_id(p, regarray_id);
10219         if (!r)
10220                 return -EINVAL;
10221
10222         strcpy(regarray->name, r->name);
10223         regarray->size = r->size;
10224         return 0;
10225 }
10226
10227 int
10228 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10229                                    const char *regarray_name,
10230                                    uint32_t regarray_index,
10231                                    uint64_t *value)
10232 {
10233         struct regarray *regarray;
10234         struct regarray_runtime *r;
10235
10236         if (!p || !regarray_name || !value)
10237                 return -EINVAL;
10238
10239         regarray = regarray_find(p, regarray_name);
10240         if (!regarray || (regarray_index >= regarray->size))
10241                 return -EINVAL;
10242
10243         r = &p->regarray_runtime[regarray->id];
10244         *value = r->regarray[regarray_index];
10245         return 0;
10246 }
10247
10248 int
10249 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10250                                    const char *regarray_name,
10251                                    uint32_t regarray_index,
10252                                    uint64_t value)
10253 {
10254         struct regarray *regarray;
10255         struct regarray_runtime *r;
10256
10257         if (!p || !regarray_name)
10258                 return -EINVAL;
10259
10260         regarray = regarray_find(p, regarray_name);
10261         if (!regarray || (regarray_index >= regarray->size))
10262                 return -EINVAL;
10263
10264         r = &p->regarray_runtime[regarray->id];
10265         r->regarray[regarray_index] = value;
10266         return 0;
10267 }
10268
10269 int
10270 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10271                               uint32_t metarray_id,
10272                               struct rte_swx_ctl_metarray_info *metarray)
10273 {
10274         struct metarray *m;
10275
10276         if (!p || !metarray)
10277                 return -EINVAL;
10278
10279         m = metarray_find_by_id(p, metarray_id);
10280         if (!m)
10281                 return -EINVAL;
10282
10283         strcpy(metarray->name, m->name);
10284         metarray->size = m->size;
10285         return 0;
10286 }
10287
10288 int
10289 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10290                               const char *name,
10291                               struct rte_meter_trtcm_params *params)
10292 {
10293         struct meter_profile *mp;
10294         int status;
10295
10296         CHECK(p, EINVAL);
10297         CHECK_NAME(name, EINVAL);
10298         CHECK(params, EINVAL);
10299         CHECK(!meter_profile_find(p, name), EEXIST);
10300
10301         /* Node allocation. */
10302         mp = calloc(1, sizeof(struct meter_profile));
10303         CHECK(mp, ENOMEM);
10304
10305         /* Node initialization. */
10306         strcpy(mp->name, name);
10307         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10308         status = rte_meter_trtcm_profile_config(&mp->profile, params);
10309         if (status) {
10310                 free(mp);
10311                 CHECK(0, EINVAL);
10312         }
10313
10314         /* Node add to tailq. */
10315         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10316
10317         return 0;
10318 }
10319
10320 int
10321 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10322                                  const char *name)
10323 {
10324         struct meter_profile *mp;
10325
10326         CHECK(p, EINVAL);
10327         CHECK_NAME(name, EINVAL);
10328
10329         mp = meter_profile_find(p, name);
10330         CHECK(mp, EINVAL);
10331         CHECK(!mp->n_users, EBUSY);
10332
10333         /* Remove node from tailq. */
10334         TAILQ_REMOVE(&p->meter_profiles, mp, node);
10335         free(mp);
10336
10337         return 0;
10338 }
10339
10340 int
10341 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10342                         const char *metarray_name,
10343                         uint32_t metarray_index)
10344 {
10345         struct meter_profile *mp_old;
10346         struct metarray *metarray;
10347         struct metarray_runtime *metarray_runtime;
10348         struct meter *m;
10349
10350         CHECK(p, EINVAL);
10351         CHECK_NAME(metarray_name, EINVAL);
10352
10353         metarray = metarray_find(p, metarray_name);
10354         CHECK(metarray, EINVAL);
10355         CHECK(metarray_index < metarray->size, EINVAL);
10356
10357         metarray_runtime = &p->metarray_runtime[metarray->id];
10358         m = &metarray_runtime->metarray[metarray_index];
10359         mp_old = m->profile;
10360
10361         meter_init(m);
10362
10363         mp_old->n_users--;
10364
10365         return 0;
10366 }
10367
10368 int
10369 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10370                       const char *metarray_name,
10371                       uint32_t metarray_index,
10372                       const char *profile_name)
10373 {
10374         struct meter_profile *mp, *mp_old;
10375         struct metarray *metarray;
10376         struct metarray_runtime *metarray_runtime;
10377         struct meter *m;
10378
10379         CHECK(p, EINVAL);
10380         CHECK_NAME(metarray_name, EINVAL);
10381
10382         metarray = metarray_find(p, metarray_name);
10383         CHECK(metarray, EINVAL);
10384         CHECK(metarray_index < metarray->size, EINVAL);
10385
10386         mp = meter_profile_find(p, profile_name);
10387         CHECK(mp, EINVAL);
10388
10389         metarray_runtime = &p->metarray_runtime[metarray->id];
10390         m = &metarray_runtime->metarray[metarray_index];
10391         mp_old = m->profile;
10392
10393         memset(m, 0, sizeof(struct meter));
10394         rte_meter_trtcm_config(&m->m, &mp->profile);
10395         m->profile = mp;
10396         m->color_mask = RTE_COLORS;
10397
10398         mp->n_users++;
10399         mp_old->n_users--;
10400
10401         return 0;
10402 }
10403
10404 int
10405 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10406                              const char *metarray_name,
10407                              uint32_t metarray_index,
10408                              struct rte_swx_ctl_meter_stats *stats)
10409 {
10410         struct metarray *metarray;
10411         struct metarray_runtime *metarray_runtime;
10412         struct meter *m;
10413
10414         CHECK(p, EINVAL);
10415         CHECK_NAME(metarray_name, EINVAL);
10416
10417         metarray = metarray_find(p, metarray_name);
10418         CHECK(metarray, EINVAL);
10419         CHECK(metarray_index < metarray->size, EINVAL);
10420
10421         CHECK(stats, EINVAL);
10422
10423         metarray_runtime = &p->metarray_runtime[metarray->id];
10424         m = &metarray_runtime->metarray[metarray_index];
10425
10426         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10427         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10428
10429         return 0;
10430 }