lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <inttypes.h>
   9 #include <sys/queue.h>
  10 #include <arpa/inet.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_prefetch.h>
  14 #include <rte_byteorder.h>
  15 #include <rte_cycles.h>
  16 #include <rte_meter.h>
  17
  18 #include <rte_swx_table_selector.h>
  19
  20 #include "rte_swx_pipeline.h"
  21 #include "rte_swx_ctl.h"
  22
  23 #define CHECK(condition, err_code)                                             \
  24 do {                                                                           \
  25         if (!(condition))                                                      \
  26                 return -(err_code);                                            \
  27 } while (0)
  28
  29 #define CHECK_NAME(name, err_code)                                             \
  30         CHECK((name) &&                                                        \
  31               (name)[0] &&                                                     \
  32               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  33               err_code)
  34
  35 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  36         CHECK((instr) &&                                                       \
  37               (instr)[0] &&                                                    \
  38               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  39                RTE_SWX_INSTRUCTION_SIZE),                                      \
  40               err_code)
  41
  42 #ifndef TRACE_LEVEL
  43 #define TRACE_LEVEL 0
  44 #endif
  45
  46 #if TRACE_LEVEL
  47 #define TRACE(...) printf(__VA_ARGS__)
  48 #else
  49 #define TRACE(...)
  50 #endif
  51
  52 /*
  53  * Environment.
  54  */
  55 #define ntoh64(x) rte_be_to_cpu_64(x)
  56 #define hton64(x) rte_cpu_to_be_64(x)
  57
  58 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  59
  60 #include <rte_malloc.h>
  61
  62 static void *
  63 env_malloc(size_t size, size_t alignment, int numa_node)
  64 {
  65         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  66 }
  67
  68 static void
  69 env_free(void *start, size_t size __rte_unused)
  70 {
  71         rte_free(start);
  72 }
  73
  74 #else
  75
  76 #include <numa.h>
  77
  78 static void *
  79 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  80 {
  81         void *start;
  82
  83         if (numa_available() == -1)
  84                 return NULL;
  85
  86         start = numa_alloc_onnode(size, numa_node);
  87         if (!start)
  88                 return NULL;
  89
  90         memset(start, 0, size);
  91         return start;
  92 }
  93
  94 static void
  95 env_free(void *start, size_t size)
  96 {
  97         if (numa_available() == -1)
  98                 return;
  99
 100         numa_free(start, size);
 101 }
 102
 103 #endif
 104
 105 /*
 106  * Struct.
 107  */
 108 struct field {
 109         char name[RTE_SWX_NAME_SIZE];
 110         uint32_t n_bits;
 111         uint32_t offset;
 112         int var_size;
 113 };
 114
 115 struct struct_type {
 116         TAILQ_ENTRY(struct_type) node;
 117         char name[RTE_SWX_NAME_SIZE];
 118         struct field *fields;
 119         uint32_t n_fields;
 120         uint32_t n_bits;
 121         uint32_t n_bits_min;
 122         int var_size;
 123 };
 124
 125 TAILQ_HEAD(struct_type_tailq, struct_type);
 126
 127 /*
 128  * Input port.
 129  */
 130 struct port_in_type {
 131         TAILQ_ENTRY(port_in_type) node;
 132         char name[RTE_SWX_NAME_SIZE];
 133         struct rte_swx_port_in_ops ops;
 134 };
 135
 136 TAILQ_HEAD(port_in_type_tailq, port_in_type);
 137
 138 struct port_in {
 139         TAILQ_ENTRY(port_in) node;
 140         struct port_in_type *type;
 141         void *obj;
 142         uint32_t id;
 143 };
 144
 145 TAILQ_HEAD(port_in_tailq, port_in);
 146
 147 struct port_in_runtime {
 148         rte_swx_port_in_pkt_rx_t pkt_rx;
 149         void *obj;
 150 };
 151
 152 /*
 153  * Output port.
 154  */
 155 struct port_out_type {
 156         TAILQ_ENTRY(port_out_type) node;
 157         char name[RTE_SWX_NAME_SIZE];
 158         struct rte_swx_port_out_ops ops;
 159 };
 160
 161 TAILQ_HEAD(port_out_type_tailq, port_out_type);
 162
 163 struct port_out {
 164         TAILQ_ENTRY(port_out) node;
 165         struct port_out_type *type;
 166         void *obj;
 167         uint32_t id;
 168 };
 169
 170 TAILQ_HEAD(port_out_tailq, port_out);
 171
 172 struct port_out_runtime {
 173         rte_swx_port_out_pkt_tx_t pkt_tx;
 174         rte_swx_port_out_flush_t flush;
 175         void *obj;
 176 };
 177
 178 /*
 179  * Extern object.
 180  */
 181 struct extern_type_member_func {
 182         TAILQ_ENTRY(extern_type_member_func) node;
 183         char name[RTE_SWX_NAME_SIZE];
 184         rte_swx_extern_type_member_func_t func;
 185         uint32_t id;
 186 };
 187
 188 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 189
 190 struct extern_type {
 191         TAILQ_ENTRY(extern_type) node;
 192         char name[RTE_SWX_NAME_SIZE];
 193         struct struct_type *mailbox_struct_type;
 194         rte_swx_extern_type_constructor_t constructor;
 195         rte_swx_extern_type_destructor_t destructor;
 196         struct extern_type_member_func_tailq funcs;
 197         uint32_t n_funcs;
 198 };
 199
 200 TAILQ_HEAD(extern_type_tailq, extern_type);
 201
 202 struct extern_obj {
 203         TAILQ_ENTRY(extern_obj) node;
 204         char name[RTE_SWX_NAME_SIZE];
 205         struct extern_type *type;
 206         void *obj;
 207         uint32_t struct_id;
 208         uint32_t id;
 209 };
 210
 211 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 212
 213 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 214 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 215 #endif
 216
 217 struct extern_obj_runtime {
 218         void *obj;
 219         uint8_t *mailbox;
 220         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 221 };
 222
 223 /*
 224  * Extern function.
 225  */
 226 struct extern_func {
 227         TAILQ_ENTRY(extern_func) node;
 228         char name[RTE_SWX_NAME_SIZE];
 229         struct struct_type *mailbox_struct_type;
 230         rte_swx_extern_func_t func;
 231         uint32_t struct_id;
 232         uint32_t id;
 233 };
 234
 235 TAILQ_HEAD(extern_func_tailq, extern_func);
 236
 237 struct extern_func_runtime {
 238         uint8_t *mailbox;
 239         rte_swx_extern_func_t func;
 240 };
 241
 242 /*
 243  * Header.
 244  */
 245 struct header {
 246         TAILQ_ENTRY(header) node;
 247         char name[RTE_SWX_NAME_SIZE];
 248         struct struct_type *st;
 249         uint32_t struct_id;
 250         uint32_t id;
 251 };
 252
 253 TAILQ_HEAD(header_tailq, header);
 254
 255 struct header_runtime {
 256         uint8_t *ptr0;
 257         uint32_t n_bytes;
 258 };
 259
 260 struct header_out_runtime {
 261         uint8_t *ptr0;
 262         uint8_t *ptr;
 263         uint32_t n_bytes;
 264 };
 265
 266 /*
 267  * Instruction.
 268  */
 269
 270 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 271  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 272  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 273  * when transferred to packet meta-data and in NBO when transferred to packet
 274  * headers.
 275  */
 276
 277 /* Notation conventions:
 278  *    -Header field: H = h.header.field (dst/src)
 279  *    -Meta-data field: M = m.field (dst/src)
 280  *    -Extern object mailbox field: E = e.field (dst/src)
 281  *    -Extern function mailbox field: F = f.field (dst/src)
 282  *    -Table action data field: T = t.field (src only)
 283  *    -Immediate value: I = 32-bit unsigned value (src only)
 284  */
 285
 286 enum instruction_type {
 287         /* rx m.port_in */
 288         INSTR_RX,
 289
 290         /* tx port_out
 291          * port_out = MI
 292          */
 293         INSTR_TX,   /* port_out = M */
 294         INSTR_TX_I, /* port_out = I */
 295
 296         /* extract h.header */
 297         INSTR_HDR_EXTRACT,
 298         INSTR_HDR_EXTRACT2,
 299         INSTR_HDR_EXTRACT3,
 300         INSTR_HDR_EXTRACT4,
 301         INSTR_HDR_EXTRACT5,
 302         INSTR_HDR_EXTRACT6,
 303         INSTR_HDR_EXTRACT7,
 304         INSTR_HDR_EXTRACT8,
 305
 306         /* emit h.header */
 307         INSTR_HDR_EMIT,
 308         INSTR_HDR_EMIT_TX,
 309         INSTR_HDR_EMIT2_TX,
 310         INSTR_HDR_EMIT3_TX,
 311         INSTR_HDR_EMIT4_TX,
 312         INSTR_HDR_EMIT5_TX,
 313         INSTR_HDR_EMIT6_TX,
 314         INSTR_HDR_EMIT7_TX,
 315         INSTR_HDR_EMIT8_TX,
 316
 317         /* validate h.header */
 318         INSTR_HDR_VALIDATE,
 319
 320         /* invalidate h.header */
 321         INSTR_HDR_INVALIDATE,
 322
 323         /* mov dst src
 324          * dst = src
 325          * dst = HMEF, src = HMEFTI
 326          */
 327         INSTR_MOV,    /* dst = MEF, src = MEFT */
 328         INSTR_MOV_MH, /* dst = MEF, src = H */
 329         INSTR_MOV_HM, /* dst = H, src = MEFT */
 330         INSTR_MOV_HH, /* dst = H, src = H */
 331         INSTR_MOV_I,  /* dst = HMEF, src = I */
 332
 333         /* dma h.header t.field
 334          * memcpy(h.header, t.field, sizeof(h.header))
 335          */
 336         INSTR_DMA_HT,
 337         INSTR_DMA_HT2,
 338         INSTR_DMA_HT3,
 339         INSTR_DMA_HT4,
 340         INSTR_DMA_HT5,
 341         INSTR_DMA_HT6,
 342         INSTR_DMA_HT7,
 343         INSTR_DMA_HT8,
 344
 345         /* add dst src
 346          * dst += src
 347          * dst = HMEF, src = HMEFTI
 348          */
 349         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 350         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 351         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 352         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 353         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 354         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 355
 356         /* sub dst src
 357          * dst -= src
 358          * dst = HMEF, src = HMEFTI
 359          */
 360         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 361         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 362         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 363         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 364         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 365         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 366
 367         /* ckadd dst src
 368          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 369          * dst = H, src = {H, h.header}
 370          */
 371         INSTR_ALU_CKADD_FIELD,    /* src = H */
 372         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 373         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 374
 375         /* cksub dst src
 376          * dst = dst '- src
 377          * dst = H, src = H
 378          */
 379         INSTR_ALU_CKSUB_FIELD,
 380
 381         /* and dst src
 382          * dst &= src
 383          * dst = HMEF, src = HMEFTI
 384          */
 385         INSTR_ALU_AND,    /* dst = MEF, src = MEFT */
 386         INSTR_ALU_AND_MH, /* dst = MEF, src = H */
 387         INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
 388         INSTR_ALU_AND_HH, /* dst = H, src = H */
 389         INSTR_ALU_AND_I,  /* dst = HMEF, src = I */
 390
 391         /* or dst src
 392          * dst |= src
 393          * dst = HMEF, src = HMEFTI
 394          */
 395         INSTR_ALU_OR,    /* dst = MEF, src = MEFT */
 396         INSTR_ALU_OR_MH, /* dst = MEF, src = H */
 397         INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
 398         INSTR_ALU_OR_HH, /* dst = H, src = H */
 399         INSTR_ALU_OR_I,  /* dst = HMEF, src = I */
 400
 401         /* xor dst src
 402          * dst ^= src
 403          * dst = HMEF, src = HMEFTI
 404          */
 405         INSTR_ALU_XOR,    /* dst = MEF, src = MEFT */
 406         INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
 407         INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
 408         INSTR_ALU_XOR_HH, /* dst = H, src = H */
 409         INSTR_ALU_XOR_I,  /* dst = HMEF, src = I */
 410
 411         /* shl dst src
 412          * dst <<= src
 413          * dst = HMEF, src = HMEFTI
 414          */
 415         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 416         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 417         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 418         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 419         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 420         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 421
 422         /* shr dst src
 423          * dst >>= src
 424          * dst = HMEF, src = HMEFTI
 425          */
 426         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 427         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 428         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 429         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 430         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 431         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 432
 433         /* regprefetch REGARRAY index
 434          * prefetch REGARRAY[index]
 435          * index = HMEFTI
 436          */
 437         INSTR_REGPREFETCH_RH, /* index = H */
 438         INSTR_REGPREFETCH_RM, /* index = MEFT */
 439         INSTR_REGPREFETCH_RI, /* index = I */
 440
 441         /* regrd dst REGARRAY index
 442          * dst = REGARRAY[index]
 443          * dst = HMEF, index = HMEFTI
 444          */
 445         INSTR_REGRD_HRH, /* dst = H, index = H */
 446         INSTR_REGRD_HRM, /* dst = H, index = MEFT */
 447         INSTR_REGRD_HRI, /* dst = H, index = I */
 448         INSTR_REGRD_MRH, /* dst = MEF, index = H */
 449         INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
 450         INSTR_REGRD_MRI, /* dst = MEF, index = I */
 451
 452         /* regwr REGARRAY index src
 453          * REGARRAY[index] = src
 454          * index = HMEFTI, src = HMEFTI
 455          */
 456         INSTR_REGWR_RHH, /* index = H, src = H */
 457         INSTR_REGWR_RHM, /* index = H, src = MEFT */
 458         INSTR_REGWR_RHI, /* index = H, src = I */
 459         INSTR_REGWR_RMH, /* index = MEFT, src = H */
 460         INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
 461         INSTR_REGWR_RMI, /* index = MEFT, src = I */
 462         INSTR_REGWR_RIH, /* index = I, src = H */
 463         INSTR_REGWR_RIM, /* index = I, src = MEFT */
 464         INSTR_REGWR_RII, /* index = I, src = I */
 465
 466         /* regadd REGARRAY index src
 467          * REGARRAY[index] += src
 468          * index = HMEFTI, src = HMEFTI
 469          */
 470         INSTR_REGADD_RHH, /* index = H, src = H */
 471         INSTR_REGADD_RHM, /* index = H, src = MEFT */
 472         INSTR_REGADD_RHI, /* index = H, src = I */
 473         INSTR_REGADD_RMH, /* index = MEFT, src = H */
 474         INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
 475         INSTR_REGADD_RMI, /* index = MEFT, src = I */
 476         INSTR_REGADD_RIH, /* index = I, src = H */
 477         INSTR_REGADD_RIM, /* index = I, src = MEFT */
 478         INSTR_REGADD_RII, /* index = I, src = I */
 479
 480         /* metprefetch METARRAY index
 481          * prefetch METARRAY[index]
 482          * index = HMEFTI
 483          */
 484         INSTR_METPREFETCH_H, /* index = H */
 485         INSTR_METPREFETCH_M, /* index = MEFT */
 486         INSTR_METPREFETCH_I, /* index = I */
 487
 488         /* meter METARRAY index length color_in color_out
 489          * color_out = meter(METARRAY[index], length, color_in)
 490          * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
 491          */
 492         INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
 493         INSTR_METER_HHI, /* index = H, length = H, color_in = I */
 494         INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
 495         INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
 496         INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
 497         INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
 498         INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
 499         INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
 500         INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
 501         INSTR_METER_IHI, /* index = I, length = H, color_in = I */
 502         INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
 503         INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
 504
 505         /* table TABLE */
 506         INSTR_TABLE,
 507         INSTR_SELECTOR,
 508
 509         /* extern e.obj.func */
 510         INSTR_EXTERN_OBJ,
 511
 512         /* extern f.func */
 513         INSTR_EXTERN_FUNC,
 514
 515         /* jmp LABEL
 516          * Unconditional jump
 517          */
 518         INSTR_JMP,
 519
 520         /* jmpv LABEL h.header
 521          * Jump if header is valid
 522          */
 523         INSTR_JMP_VALID,
 524
 525         /* jmpnv LABEL h.header
 526          * Jump if header is invalid
 527          */
 528         INSTR_JMP_INVALID,
 529
 530         /* jmph LABEL
 531          * Jump if table lookup hit
 532          */
 533         INSTR_JMP_HIT,
 534
 535         /* jmpnh LABEL
 536          * Jump if table lookup miss
 537          */
 538         INSTR_JMP_MISS,
 539
 540         /* jmpa LABEL ACTION
 541          * Jump if action run
 542          */
 543         INSTR_JMP_ACTION_HIT,
 544
 545         /* jmpna LABEL ACTION
 546          * Jump if action not run
 547          */
 548         INSTR_JMP_ACTION_MISS,
 549
 550         /* jmpeq LABEL a b
 551          * Jump if a is equal to b
 552          * a = HMEFT, b = HMEFTI
 553          */
 554         INSTR_JMP_EQ,    /* a = MEFT, b = MEFT */
 555         INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
 556         INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
 557         INSTR_JMP_EQ_HH, /* a = H, b = H */
 558         INSTR_JMP_EQ_I,  /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 559
 560         /* jmpneq LABEL a b
 561          * Jump if a is not equal to b
 562          * a = HMEFT, b = HMEFTI
 563          */
 564         INSTR_JMP_NEQ,    /* a = MEFT, b = MEFT */
 565         INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
 566         INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
 567         INSTR_JMP_NEQ_HH, /* a = H, b = H */
 568         INSTR_JMP_NEQ_I,  /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 569
 570         /* jmplt LABEL a b
 571          * Jump if a is less than b
 572          * a = HMEFT, b = HMEFTI
 573          */
 574         INSTR_JMP_LT,    /* a = MEFT, b = MEFT */
 575         INSTR_JMP_LT_MH, /* a = MEFT, b = H */
 576         INSTR_JMP_LT_HM, /* a = H, b = MEFT */
 577         INSTR_JMP_LT_HH, /* a = H, b = H */
 578         INSTR_JMP_LT_MI, /* a = MEFT, b = I */
 579         INSTR_JMP_LT_HI, /* a = H, b = I */
 580
 581         /* jmpgt LABEL a b
 582          * Jump if a is greater than b
 583          * a = HMEFT, b = HMEFTI
 584          */
 585         INSTR_JMP_GT,    /* a = MEFT, b = MEFT */
 586         INSTR_JMP_GT_MH, /* a = MEFT, b = H */
 587         INSTR_JMP_GT_HM, /* a = H, b = MEFT */
 588         INSTR_JMP_GT_HH, /* a = H, b = H */
 589         INSTR_JMP_GT_MI, /* a = MEFT, b = I */
 590         INSTR_JMP_GT_HI, /* a = H, b = I */
 591
 592         /* return
 593          * Return from action
 594          */
 595         INSTR_RETURN,
 596 };
 597
 598 struct instr_operand {
 599         uint8_t struct_id;
 600         uint8_t n_bits;
 601         uint8_t offset;
 602         uint8_t pad;
 603 };
 604
 605 struct instr_io {
 606         struct {
 607                 union {
 608                         struct {
 609                                 uint8_t offset;
 610                                 uint8_t n_bits;
 611                                 uint8_t pad[2];
 612                         };
 613
 614                         uint32_t val;
 615                 };
 616         } io;
 617
 618         struct {
 619                 uint8_t header_id[8];
 620                 uint8_t struct_id[8];
 621                 uint8_t n_bytes[8];
 622         } hdr;
 623 };
 624
 625 struct instr_hdr_validity {
 626         uint8_t header_id;
 627 };
 628
 629 struct instr_table {
 630         uint8_t table_id;
 631 };
 632
 633 struct instr_extern_obj {
 634         uint8_t ext_obj_id;
 635         uint8_t func_id;
 636 };
 637
 638 struct instr_extern_func {
 639         uint8_t ext_func_id;
 640 };
 641
 642 struct instr_dst_src {
 643         struct instr_operand dst;
 644         union {
 645                 struct instr_operand src;
 646                 uint64_t src_val;
 647         };
 648 };
 649
 650 struct instr_regarray {
 651         uint8_t regarray_id;
 652         uint8_t pad[3];
 653
 654         union {
 655                 struct instr_operand idx;
 656                 uint32_t idx_val;
 657         };
 658
 659         union {
 660                 struct instr_operand dstsrc;
 661                 uint64_t dstsrc_val;
 662         };
 663 };
 664
 665 struct instr_meter {
 666         uint8_t metarray_id;
 667         uint8_t pad[3];
 668
 669         union {
 670                 struct instr_operand idx;
 671                 uint32_t idx_val;
 672         };
 673
 674         struct instr_operand length;
 675
 676         union {
 677                 struct instr_operand color_in;
 678                 uint32_t color_in_val;
 679         };
 680
 681         struct instr_operand color_out;
 682 };
 683
 684 struct instr_dma {
 685         struct {
 686                 uint8_t header_id[8];
 687                 uint8_t struct_id[8];
 688         } dst;
 689
 690         struct {
 691                 uint8_t offset[8];
 692         } src;
 693
 694         uint16_t n_bytes[8];
 695 };
 696
 697 struct instr_jmp {
 698         struct instruction *ip;
 699
 700         union {
 701                 struct instr_operand a;
 702                 uint8_t header_id;
 703                 uint8_t action_id;
 704         };
 705
 706         union {
 707                 struct instr_operand b;
 708                 uint64_t b_val;
 709         };
 710 };
 711
 712 struct instruction {
 713         enum instruction_type type;
 714         union {
 715                 struct instr_io io;
 716                 struct instr_hdr_validity valid;
 717                 struct instr_dst_src mov;
 718                 struct instr_regarray regarray;
 719                 struct instr_meter meter;
 720                 struct instr_dma dma;
 721                 struct instr_dst_src alu;
 722                 struct instr_table table;
 723                 struct instr_extern_obj ext_obj;
 724                 struct instr_extern_func ext_func;
 725                 struct instr_jmp jmp;
 726         };
 727 };
 728
 729 struct instruction_data {
 730         char label[RTE_SWX_NAME_SIZE];
 731         char jmp_label[RTE_SWX_NAME_SIZE];
 732         uint32_t n_users; /* user = jmp instruction to this instruction. */
 733         int invalid;
 734 };
 735
 736 /*
 737  * Action.
 738  */
 739 struct action {
 740         TAILQ_ENTRY(action) node;
 741         char name[RTE_SWX_NAME_SIZE];
 742         struct struct_type *st;
 743         int *args_endianness; /* 0 = Host Byte Order (HBO). */
 744         struct instruction *instructions;
 745         uint32_t n_instructions;
 746         uint32_t id;
 747 };
 748
 749 TAILQ_HEAD(action_tailq, action);
 750
 751 /*
 752  * Table.
 753  */
 754 struct table_type {
 755         TAILQ_ENTRY(table_type) node;
 756         char name[RTE_SWX_NAME_SIZE];
 757         enum rte_swx_table_match_type match_type;
 758         struct rte_swx_table_ops ops;
 759 };
 760
 761 TAILQ_HEAD(table_type_tailq, table_type);
 762
 763 struct match_field {
 764         enum rte_swx_table_match_type match_type;
 765         struct field *field;
 766 };
 767
 768 struct table {
 769         TAILQ_ENTRY(table) node;
 770         char name[RTE_SWX_NAME_SIZE];
 771         char args[RTE_SWX_NAME_SIZE];
 772         struct table_type *type; /* NULL when n_fields == 0. */
 773
 774         /* Match. */
 775         struct match_field *fields;
 776         uint32_t n_fields;
 777         struct header *header; /* Only valid when n_fields > 0. */
 778
 779         /* Action. */
 780         struct action **actions;
 781         struct action *default_action;
 782         uint8_t *default_action_data;
 783         uint32_t n_actions;
 784         int default_action_is_const;
 785         uint32_t action_data_size_max;
 786
 787         uint32_t size;
 788         uint32_t id;
 789 };
 790
 791 TAILQ_HEAD(table_tailq, table);
 792
 793 struct table_runtime {
 794         rte_swx_table_lookup_t func;
 795         void *mailbox;
 796         uint8_t **key;
 797 };
 798
 799 struct table_statistics {
 800         uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
 801         uint64_t *n_pkts_action;
 802 };
 803
 804 /*
 805  * Selector.
 806  */
 807 struct selector {
 808         TAILQ_ENTRY(selector) node;
 809         char name[RTE_SWX_NAME_SIZE];
 810
 811         struct field *group_id_field;
 812         struct field **selector_fields;
 813         uint32_t n_selector_fields;
 814         struct header *selector_header;
 815         struct field *member_id_field;
 816
 817         uint32_t n_groups_max;
 818         uint32_t n_members_per_group_max;
 819
 820         uint32_t id;
 821 };
 822
 823 TAILQ_HEAD(selector_tailq, selector);
 824
 825 struct selector_runtime {
 826         void *mailbox;
 827         uint8_t **group_id_buffer;
 828         uint8_t **selector_buffer;
 829         uint8_t **member_id_buffer;
 830 };
 831
 832 struct selector_statistics {
 833         uint64_t n_pkts;
 834 };
 835
 836 /*
 837  * Register array.
 838  */
 839 struct regarray {
 840         TAILQ_ENTRY(regarray) node;
 841         char name[RTE_SWX_NAME_SIZE];
 842         uint64_t init_val;
 843         uint32_t size;
 844         uint32_t id;
 845 };
 846
 847 TAILQ_HEAD(regarray_tailq, regarray);
 848
 849 struct regarray_runtime {
 850         uint64_t *regarray;
 851         uint32_t size_mask;
 852 };
 853
 854 /*
 855  * Meter array.
 856  */
 857 struct meter_profile {
 858         TAILQ_ENTRY(meter_profile) node;
 859         char name[RTE_SWX_NAME_SIZE];
 860         struct rte_meter_trtcm_params params;
 861         struct rte_meter_trtcm_profile profile;
 862         uint32_t n_users;
 863 };
 864
 865 TAILQ_HEAD(meter_profile_tailq, meter_profile);
 866
 867 struct metarray {
 868         TAILQ_ENTRY(metarray) node;
 869         char name[RTE_SWX_NAME_SIZE];
 870         uint32_t size;
 871         uint32_t id;
 872 };
 873
 874 TAILQ_HEAD(metarray_tailq, metarray);
 875
 876 struct meter {
 877         struct rte_meter_trtcm m;
 878         struct meter_profile *profile;
 879         enum rte_color color_mask;
 880         uint8_t pad[20];
 881
 882         uint64_t n_pkts[RTE_COLORS];
 883         uint64_t n_bytes[RTE_COLORS];
 884 };
 885
 886 struct metarray_runtime {
 887         struct meter *metarray;
 888         uint32_t size_mask;
 889 };
 890
 891 /*
 892  * Pipeline.
 893  */
 894 struct thread {
 895         /* Packet. */
 896         struct rte_swx_pkt pkt;
 897         uint8_t *ptr;
 898
 899         /* Structures. */
 900         uint8_t **structs;
 901
 902         /* Packet headers. */
 903         struct header_runtime *headers; /* Extracted or generated headers. */
 904         struct header_out_runtime *headers_out; /* Emitted headers. */
 905         uint8_t *header_storage;
 906         uint8_t *header_out_storage;
 907         uint64_t valid_headers;
 908         uint32_t n_headers_out;
 909
 910         /* Packet meta-data. */
 911         uint8_t *metadata;
 912
 913         /* Tables. */
 914         struct table_runtime *tables;
 915         struct selector_runtime *selectors;
 916         struct rte_swx_table_state *table_state;
 917         uint64_t action_id;
 918         int hit; /* 0 = Miss, 1 = Hit. */
 919
 920         /* Extern objects and functions. */
 921         struct extern_obj_runtime *extern_objs;
 922         struct extern_func_runtime *extern_funcs;
 923
 924         /* Instructions. */
 925         struct instruction *ip;
 926         struct instruction *ret;
 927 };
 928
 929 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 930 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 931 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 932
 933 #define HEADER_VALID(thread, header_id) \
 934         MASK64_BIT_GET((thread)->valid_headers, header_id)
 935
 936 #define ALU(thread, ip, operator)  \
 937 {                                                                              \
 938         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 939         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 940         uint64_t dst64 = *dst64_ptr;                                           \
 941         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 942         uint64_t dst = dst64 & dst64_mask;                                     \
 943                                                                                \
 944         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 945         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 946         uint64_t src64 = *src64_ptr;                                           \
 947         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 948         uint64_t src = src64 & src64_mask;                                     \
 949                                                                                \
 950         uint64_t result = dst operator src;                                    \
 951                                                                                \
 952         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 953 }
 954
 955 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 956
 957 #define ALU_MH(thread, ip, operator)  \
 958 {                                                                              \
 959         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 960         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 961         uint64_t dst64 = *dst64_ptr;                                           \
 962         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 963         uint64_t dst = dst64 & dst64_mask;                                     \
 964                                                                                \
 965         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 966         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 967         uint64_t src64 = *src64_ptr;                                           \
 968         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 969                                                                                \
 970         uint64_t result = dst operator src;                                    \
 971                                                                                \
 972         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 973 }
 974
 975 #define ALU_HM(thread, ip, operator)  \
 976 {                                                                              \
 977         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 978         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 979         uint64_t dst64 = *dst64_ptr;                                           \
 980         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 981         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 982                                                                                \
 983         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 984         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 985         uint64_t src64 = *src64_ptr;                                           \
 986         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 987         uint64_t src = src64 & src64_mask;                                     \
 988                                                                                \
 989         uint64_t result = dst operator src;                                    \
 990         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 991                                                                                \
 992         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 993 }
 994
 995 #define ALU_HM_FAST(thread, ip, operator)  \
 996 {                                                                                 \
 997         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];         \
 998         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];      \
 999         uint64_t dst64 = *dst64_ptr;                                              \
1000         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);          \
1001         uint64_t dst = dst64 & dst64_mask;                                        \
1002                                                                                   \
1003         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];         \
1004         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];      \
1005         uint64_t src64 = *src64_ptr;                                              \
1006         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);          \
1007         uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1008                                                                                   \
1009         uint64_t result = dst operator src;                                       \
1010                                                                                   \
1011         *dst64_ptr = (dst64 & ~dst64_mask) | result;                              \
1012 }
1013
1014 #define ALU_HH(thread, ip, operator)  \
1015 {                                                                              \
1016         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1017         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1018         uint64_t dst64 = *dst64_ptr;                                           \
1019         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1020         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
1021                                                                                \
1022         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
1023         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
1024         uint64_t src64 = *src64_ptr;                                           \
1025         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
1026                                                                                \
1027         uint64_t result = dst operator src;                                    \
1028         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1029                                                                                \
1030         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1031 }
1032
1033 #define ALU_HH_FAST(thread, ip, operator)  \
1034 {                                                                                             \
1035         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];                     \
1036         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];                  \
1037         uint64_t dst64 = *dst64_ptr;                                                          \
1038         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);                      \
1039         uint64_t dst = dst64 & dst64_mask;                                                    \
1040                                                                                               \
1041         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];                     \
1042         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];                  \
1043         uint64_t src64 = *src64_ptr;                                                          \
1044         uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1045                                                                                               \
1046         uint64_t result = dst operator src;                                                   \
1047                                                                                               \
1048         *dst64_ptr = (dst64 & ~dst64_mask) | result;                                          \
1049 }
1050
1051 #else
1052
1053 #define ALU_MH ALU
1054 #define ALU_HM ALU
1055 #define ALU_HM_FAST ALU
1056 #define ALU_HH ALU
1057 #define ALU_HH_FAST ALU
1058
1059 #endif
1060
1061 #define ALU_I(thread, ip, operator)  \
1062 {                                                                              \
1063         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1064         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1065         uint64_t dst64 = *dst64_ptr;                                           \
1066         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1067         uint64_t dst = dst64 & dst64_mask;                                     \
1068                                                                                \
1069         uint64_t src = (ip)->alu.src_val;                                      \
1070                                                                                \
1071         uint64_t result = dst operator src;                                    \
1072                                                                                \
1073         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
1074 }
1075
1076 #define ALU_MI ALU_I
1077
1078 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1079
1080 #define ALU_HI(thread, ip, operator)  \
1081 {                                                                              \
1082         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1083         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1084         uint64_t dst64 = *dst64_ptr;                                           \
1085         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1086         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
1087                                                                                \
1088         uint64_t src = (ip)->alu.src_val;                                      \
1089                                                                                \
1090         uint64_t result = dst operator src;                                    \
1091         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1092                                                                                \
1093         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1094 }
1095
1096 #else
1097
1098 #define ALU_HI ALU_I
1099
1100 #endif
1101
1102 #define MOV(thread, ip)  \
1103 {                                                                              \
1104         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1105         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1106         uint64_t dst64 = *dst64_ptr;                                           \
1107         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1108                                                                                \
1109         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1110         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1111         uint64_t src64 = *src64_ptr;                                           \
1112         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1113         uint64_t src = src64 & src64_mask;                                     \
1114                                                                                \
1115         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1116 }
1117
1118 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1119
1120 #define MOV_MH(thread, ip)  \
1121 {                                                                              \
1122         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1123         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1124         uint64_t dst64 = *dst64_ptr;                                           \
1125         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1126                                                                                \
1127         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1128         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1129         uint64_t src64 = *src64_ptr;                                           \
1130         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
1131                                                                                \
1132         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1133 }
1134
1135 #define MOV_HM(thread, ip)  \
1136 {                                                                              \
1137         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1138         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1139         uint64_t dst64 = *dst64_ptr;                                           \
1140         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1141                                                                                \
1142         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1143         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1144         uint64_t src64 = *src64_ptr;                                           \
1145         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1146         uint64_t src = src64 & src64_mask;                                     \
1147                                                                                \
1148         src = hton64(src) >> (64 - (ip)->mov.dst.n_bits);                      \
1149         *dst64_ptr = (dst64 & ~dst64_mask) | src;                              \
1150 }
1151
1152 #define MOV_HH(thread, ip)  \
1153 {                                                                              \
1154         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1155         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1156         uint64_t dst64 = *dst64_ptr;                                           \
1157         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1158                                                                                \
1159         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1160         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1161         uint64_t src64 = *src64_ptr;                                           \
1162                                                                                \
1163         uint64_t src = src64 << (64 - (ip)->mov.src.n_bits);                   \
1164         src = src >> (64 - (ip)->mov.dst.n_bits);                              \
1165         *dst64_ptr = (dst64 & ~dst64_mask) | src;                              \
1166 }
1167
1168 #else
1169
1170 #define MOV_MH MOV
1171 #define MOV_HM MOV
1172 #define MOV_HH MOV
1173
1174 #endif
1175
1176 #define MOV_I(thread, ip)  \
1177 {                                                                              \
1178         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1179         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1180         uint64_t dst64 = *dst64_ptr;                                           \
1181         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1182                                                                                \
1183         uint64_t src = (ip)->mov.src_val;                                      \
1184                                                                                \
1185         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1186 }
1187
1188 #define JMP_CMP(thread, ip, operator)  \
1189 {                                                                              \
1190         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1191         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1192         uint64_t a64 = *a64_ptr;                                               \
1193         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1194         uint64_t a = a64 & a64_mask;                                           \
1195                                                                                \
1196         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1197         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1198         uint64_t b64 = *b64_ptr;                                               \
1199         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1200         uint64_t b = b64 & b64_mask;                                           \
1201                                                                                \
1202         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1203 }
1204
1205 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1206
1207 #define JMP_CMP_MH(thread, ip, operator)  \
1208 {                                                                              \
1209         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1210         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1211         uint64_t a64 = *a64_ptr;                                               \
1212         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1213         uint64_t a = a64 & a64_mask;                                           \
1214                                                                                \
1215         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1216         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1217         uint64_t b64 = *b64_ptr;                                               \
1218         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1219                                                                                \
1220         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1221 }
1222
1223 #define JMP_CMP_HM(thread, ip, operator)  \
1224 {                                                                              \
1225         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1226         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1227         uint64_t a64 = *a64_ptr;                                               \
1228         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1229                                                                                \
1230         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1231         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1232         uint64_t b64 = *b64_ptr;                                               \
1233         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1234         uint64_t b = b64 & b64_mask;                                           \
1235                                                                                \
1236         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1237 }
1238
1239 #define JMP_CMP_HH(thread, ip, operator)  \
1240 {                                                                              \
1241         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1242         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1243         uint64_t a64 = *a64_ptr;                                               \
1244         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1245                                                                                \
1246         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1247         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1248         uint64_t b64 = *b64_ptr;                                               \
1249         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1250                                                                                \
1251         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1252 }
1253
1254 #define JMP_CMP_HH_FAST(thread, ip, operator)  \
1255 {                                                                              \
1256         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1257         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1258         uint64_t a64 = *a64_ptr;                                               \
1259         uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits);                         \
1260                                                                                \
1261         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1262         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1263         uint64_t b64 = *b64_ptr;                                               \
1264         uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits);                         \
1265                                                                                \
1266         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1267 }
1268
1269 #else
1270
1271 #define JMP_CMP_MH JMP_CMP
1272 #define JMP_CMP_HM JMP_CMP
1273 #define JMP_CMP_HH JMP_CMP
1274 #define JMP_CMP_HH_FAST JMP_CMP
1275
1276 #endif
1277
1278 #define JMP_CMP_I(thread, ip, operator)  \
1279 {                                                                              \
1280         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1281         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1282         uint64_t a64 = *a64_ptr;                                               \
1283         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1284         uint64_t a = a64 & a64_mask;                                           \
1285                                                                                \
1286         uint64_t b = (ip)->jmp.b_val;                                          \
1287                                                                                \
1288         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1289 }
1290
1291 #define JMP_CMP_MI JMP_CMP_I
1292
1293 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1294
1295 #define JMP_CMP_HI(thread, ip, operator)  \
1296 {                                                                              \
1297         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1298         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1299         uint64_t a64 = *a64_ptr;                                               \
1300         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1301                                                                                \
1302         uint64_t b = (ip)->jmp.b_val;                                          \
1303                                                                                \
1304         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1305 }
1306
1307 #else
1308
1309 #define JMP_CMP_HI JMP_CMP_I
1310
1311 #endif
1312
1313 #define METADATA_READ(thread, offset, n_bits)                                  \
1314 ({                                                                             \
1315         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1316         uint64_t m64 = *m64_ptr;                                               \
1317         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1318         (m64 & m64_mask);                                                      \
1319 })
1320
1321 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
1322 {                                                                              \
1323         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1324         uint64_t m64 = *m64_ptr;                                               \
1325         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1326                                                                                \
1327         uint64_t m_new = value;                                                \
1328                                                                                \
1329         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
1330 }
1331
1332 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1333 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1334 #endif
1335
1336 struct rte_swx_pipeline {
1337         struct struct_type_tailq struct_types;
1338         struct port_in_type_tailq port_in_types;
1339         struct port_in_tailq ports_in;
1340         struct port_out_type_tailq port_out_types;
1341         struct port_out_tailq ports_out;
1342         struct extern_type_tailq extern_types;
1343         struct extern_obj_tailq extern_objs;
1344         struct extern_func_tailq extern_funcs;
1345         struct header_tailq headers;
1346         struct struct_type *metadata_st;
1347         uint32_t metadata_struct_id;
1348         struct action_tailq actions;
1349         struct table_type_tailq table_types;
1350         struct table_tailq tables;
1351         struct selector_tailq selectors;
1352         struct regarray_tailq regarrays;
1353         struct meter_profile_tailq meter_profiles;
1354         struct metarray_tailq metarrays;
1355
1356         struct port_in_runtime *in;
1357         struct port_out_runtime *out;
1358         struct instruction **action_instructions;
1359         struct rte_swx_table_state *table_state;
1360         struct table_statistics *table_stats;
1361         struct selector_statistics *selector_stats;
1362         struct regarray_runtime *regarray_runtime;
1363         struct metarray_runtime *metarray_runtime;
1364         struct instruction *instructions;
1365         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1366
1367         uint32_t n_structs;
1368         uint32_t n_ports_in;
1369         uint32_t n_ports_out;
1370         uint32_t n_extern_objs;
1371         uint32_t n_extern_funcs;
1372         uint32_t n_actions;
1373         uint32_t n_tables;
1374         uint32_t n_selectors;
1375         uint32_t n_regarrays;
1376         uint32_t n_metarrays;
1377         uint32_t n_headers;
1378         uint32_t thread_id;
1379         uint32_t port_id;
1380         uint32_t n_instructions;
1381         int build_done;
1382         int numa_node;
1383 };
1384
1385 /*
1386  * Struct.
1387  */
1388 static struct struct_type *
1389 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1390 {
1391         struct struct_type *elem;
1392
1393         TAILQ_FOREACH(elem, &p->struct_types, node)
1394                 if (strcmp(elem->name, name) == 0)
1395                         return elem;
1396
1397         return NULL;
1398 }
1399
1400 static struct field *
1401 struct_type_field_find(struct struct_type *st, const char *name)
1402 {
1403         uint32_t i;
1404
1405         for (i = 0; i < st->n_fields; i++) {
1406                 struct field *f = &st->fields[i];
1407
1408                 if (strcmp(f->name, name) == 0)
1409                         return f;
1410         }
1411
1412         return NULL;
1413 }
1414
1415 int
1416 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1417                                       const char *name,
1418                                       struct rte_swx_field_params *fields,
1419                                       uint32_t n_fields,
1420                                       int last_field_has_variable_size)
1421 {
1422         struct struct_type *st;
1423         uint32_t i;
1424
1425         CHECK(p, EINVAL);
1426         CHECK_NAME(name, EINVAL);
1427         CHECK(fields, EINVAL);
1428         CHECK(n_fields, EINVAL);
1429
1430         for (i = 0; i < n_fields; i++) {
1431                 struct rte_swx_field_params *f = &fields[i];
1432                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
1433                 uint32_t j;
1434
1435                 CHECK_NAME(f->name, EINVAL);
1436                 CHECK(f->n_bits, EINVAL);
1437                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
1438                 CHECK((f->n_bits & 7) == 0, EINVAL);
1439
1440                 for (j = 0; j < i; j++) {
1441                         struct rte_swx_field_params *f_prev = &fields[j];
1442
1443                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
1444                 }
1445         }
1446
1447         CHECK(!struct_type_find(p, name), EEXIST);
1448
1449         /* Node allocation. */
1450         st = calloc(1, sizeof(struct struct_type));
1451         CHECK(st, ENOMEM);
1452
1453         st->fields = calloc(n_fields, sizeof(struct field));
1454         if (!st->fields) {
1455                 free(st);
1456                 CHECK(0, ENOMEM);
1457         }
1458
1459         /* Node initialization. */
1460         strcpy(st->name, name);
1461         for (i = 0; i < n_fields; i++) {
1462                 struct field *dst = &st->fields[i];
1463                 struct rte_swx_field_params *src = &fields[i];
1464                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
1465
1466                 strcpy(dst->name, src->name);
1467                 dst->n_bits = src->n_bits;
1468                 dst->offset = st->n_bits;
1469                 dst->var_size = var_size;
1470
1471                 st->n_bits += src->n_bits;
1472                 st->n_bits_min += var_size ? 0 : src->n_bits;
1473         }
1474         st->n_fields = n_fields;
1475         st->var_size = last_field_has_variable_size;
1476
1477         /* Node add to tailq. */
1478         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1479
1480         return 0;
1481 }
1482
1483 static int
1484 struct_build(struct rte_swx_pipeline *p)
1485 {
1486         uint32_t i;
1487
1488         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1489                 struct thread *t = &p->threads[i];
1490
1491                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1492                 CHECK(t->structs, ENOMEM);
1493         }
1494
1495         return 0;
1496 }
1497
1498 static void
1499 struct_build_free(struct rte_swx_pipeline *p)
1500 {
1501         uint32_t i;
1502
1503         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1504                 struct thread *t = &p->threads[i];
1505
1506                 free(t->structs);
1507                 t->structs = NULL;
1508         }
1509 }
1510
1511 static void
1512 struct_free(struct rte_swx_pipeline *p)
1513 {
1514         struct_build_free(p);
1515
1516         /* Struct types. */
1517         for ( ; ; ) {
1518                 struct struct_type *elem;
1519
1520                 elem = TAILQ_FIRST(&p->struct_types);
1521                 if (!elem)
1522                         break;
1523
1524                 TAILQ_REMOVE(&p->struct_types, elem, node);
1525                 free(elem->fields);
1526                 free(elem);
1527         }
1528 }
1529
1530 /*
1531  * Input port.
1532  */
1533 static struct port_in_type *
1534 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1535 {
1536         struct port_in_type *elem;
1537
1538         if (!name)
1539                 return NULL;
1540
1541         TAILQ_FOREACH(elem, &p->port_in_types, node)
1542                 if (strcmp(elem->name, name) == 0)
1543                         return elem;
1544
1545         return NULL;
1546 }
1547
1548 int
1549 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1550                                        const char *name,
1551                                        struct rte_swx_port_in_ops *ops)
1552 {
1553         struct port_in_type *elem;
1554
1555         CHECK(p, EINVAL);
1556         CHECK_NAME(name, EINVAL);
1557         CHECK(ops, EINVAL);
1558         CHECK(ops->create, EINVAL);
1559         CHECK(ops->free, EINVAL);
1560         CHECK(ops->pkt_rx, EINVAL);
1561         CHECK(ops->stats_read, EINVAL);
1562
1563         CHECK(!port_in_type_find(p, name), EEXIST);
1564
1565         /* Node allocation. */
1566         elem = calloc(1, sizeof(struct port_in_type));
1567         CHECK(elem, ENOMEM);
1568
1569         /* Node initialization. */
1570         strcpy(elem->name, name);
1571         memcpy(&elem->ops, ops, sizeof(*ops));
1572
1573         /* Node add to tailq. */
1574         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1575
1576         return 0;
1577 }
1578
1579 static struct port_in *
1580 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1581 {
1582         struct port_in *port;
1583
1584         TAILQ_FOREACH(port, &p->ports_in, node)
1585                 if (port->id == port_id)
1586                         return port;
1587
1588         return NULL;
1589 }
1590
1591 int
1592 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1593                                 uint32_t port_id,
1594                                 const char *port_type_name,
1595                                 void *args)
1596 {
1597         struct port_in_type *type = NULL;
1598         struct port_in *port = NULL;
1599         void *obj = NULL;
1600
1601         CHECK(p, EINVAL);
1602
1603         CHECK(!port_in_find(p, port_id), EINVAL);
1604
1605         CHECK_NAME(port_type_name, EINVAL);
1606         type = port_in_type_find(p, port_type_name);
1607         CHECK(type, EINVAL);
1608
1609         obj = type->ops.create(args);
1610         CHECK(obj, ENODEV);
1611
1612         /* Node allocation. */
1613         port = calloc(1, sizeof(struct port_in));
1614         CHECK(port, ENOMEM);
1615
1616         /* Node initialization. */
1617         port->type = type;
1618         port->obj = obj;
1619         port->id = port_id;
1620
1621         /* Node add to tailq. */
1622         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1623         if (p->n_ports_in < port_id + 1)
1624                 p->n_ports_in = port_id + 1;
1625
1626         return 0;
1627 }
1628
1629 static int
1630 port_in_build(struct rte_swx_pipeline *p)
1631 {
1632         struct port_in *port;
1633         uint32_t i;
1634
1635         CHECK(p->n_ports_in, EINVAL);
1636         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1637
1638         for (i = 0; i < p->n_ports_in; i++)
1639                 CHECK(port_in_find(p, i), EINVAL);
1640
1641         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1642         CHECK(p->in, ENOMEM);
1643
1644         TAILQ_FOREACH(port, &p->ports_in, node) {
1645                 struct port_in_runtime *in = &p->in[port->id];
1646
1647                 in->pkt_rx = port->type->ops.pkt_rx;
1648                 in->obj = port->obj;
1649         }
1650
1651         return 0;
1652 }
1653
1654 static void
1655 port_in_build_free(struct rte_swx_pipeline *p)
1656 {
1657         free(p->in);
1658         p->in = NULL;
1659 }
1660
1661 static void
1662 port_in_free(struct rte_swx_pipeline *p)
1663 {
1664         port_in_build_free(p);
1665
1666         /* Input ports. */
1667         for ( ; ; ) {
1668                 struct port_in *port;
1669
1670                 port = TAILQ_FIRST(&p->ports_in);
1671                 if (!port)
1672                         break;
1673
1674                 TAILQ_REMOVE(&p->ports_in, port, node);
1675                 port->type->ops.free(port->obj);
1676                 free(port);
1677         }
1678
1679         /* Input port types. */
1680         for ( ; ; ) {
1681                 struct port_in_type *elem;
1682
1683                 elem = TAILQ_FIRST(&p->port_in_types);
1684                 if (!elem)
1685                         break;
1686
1687                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1688                 free(elem);
1689         }
1690 }
1691
1692 /*
1693  * Output port.
1694  */
1695 static struct port_out_type *
1696 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1697 {
1698         struct port_out_type *elem;
1699
1700         if (!name)
1701                 return NULL;
1702
1703         TAILQ_FOREACH(elem, &p->port_out_types, node)
1704                 if (!strcmp(elem->name, name))
1705                         return elem;
1706
1707         return NULL;
1708 }
1709
1710 int
1711 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1712                                         const char *name,
1713                                         struct rte_swx_port_out_ops *ops)
1714 {
1715         struct port_out_type *elem;
1716
1717         CHECK(p, EINVAL);
1718         CHECK_NAME(name, EINVAL);
1719         CHECK(ops, EINVAL);
1720         CHECK(ops->create, EINVAL);
1721         CHECK(ops->free, EINVAL);
1722         CHECK(ops->pkt_tx, EINVAL);
1723         CHECK(ops->stats_read, EINVAL);
1724
1725         CHECK(!port_out_type_find(p, name), EEXIST);
1726
1727         /* Node allocation. */
1728         elem = calloc(1, sizeof(struct port_out_type));
1729         CHECK(elem, ENOMEM);
1730
1731         /* Node initialization. */
1732         strcpy(elem->name, name);
1733         memcpy(&elem->ops, ops, sizeof(*ops));
1734
1735         /* Node add to tailq. */
1736         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1737
1738         return 0;
1739 }
1740
1741 static struct port_out *
1742 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1743 {
1744         struct port_out *port;
1745
1746         TAILQ_FOREACH(port, &p->ports_out, node)
1747                 if (port->id == port_id)
1748                         return port;
1749
1750         return NULL;
1751 }
1752
1753 int
1754 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1755                                  uint32_t port_id,
1756                                  const char *port_type_name,
1757                                  void *args)
1758 {
1759         struct port_out_type *type = NULL;
1760         struct port_out *port = NULL;
1761         void *obj = NULL;
1762
1763         CHECK(p, EINVAL);
1764
1765         CHECK(!port_out_find(p, port_id), EINVAL);
1766
1767         CHECK_NAME(port_type_name, EINVAL);
1768         type = port_out_type_find(p, port_type_name);
1769         CHECK(type, EINVAL);
1770
1771         obj = type->ops.create(args);
1772         CHECK(obj, ENODEV);
1773
1774         /* Node allocation. */
1775         port = calloc(1, sizeof(struct port_out));
1776         CHECK(port, ENOMEM);
1777
1778         /* Node initialization. */
1779         port->type = type;
1780         port->obj = obj;
1781         port->id = port_id;
1782
1783         /* Node add to tailq. */
1784         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1785         if (p->n_ports_out < port_id + 1)
1786                 p->n_ports_out = port_id + 1;
1787
1788         return 0;
1789 }
1790
1791 static int
1792 port_out_build(struct rte_swx_pipeline *p)
1793 {
1794         struct port_out *port;
1795         uint32_t i;
1796
1797         CHECK(p->n_ports_out, EINVAL);
1798
1799         for (i = 0; i < p->n_ports_out; i++)
1800                 CHECK(port_out_find(p, i), EINVAL);
1801
1802         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1803         CHECK(p->out, ENOMEM);
1804
1805         TAILQ_FOREACH(port, &p->ports_out, node) {
1806                 struct port_out_runtime *out = &p->out[port->id];
1807
1808                 out->pkt_tx = port->type->ops.pkt_tx;
1809                 out->flush = port->type->ops.flush;
1810                 out->obj = port->obj;
1811         }
1812
1813         return 0;
1814 }
1815
1816 static void
1817 port_out_build_free(struct rte_swx_pipeline *p)
1818 {
1819         free(p->out);
1820         p->out = NULL;
1821 }
1822
1823 static void
1824 port_out_free(struct rte_swx_pipeline *p)
1825 {
1826         port_out_build_free(p);
1827
1828         /* Output ports. */
1829         for ( ; ; ) {
1830                 struct port_out *port;
1831
1832                 port = TAILQ_FIRST(&p->ports_out);
1833                 if (!port)
1834                         break;
1835
1836                 TAILQ_REMOVE(&p->ports_out, port, node);
1837                 port->type->ops.free(port->obj);
1838                 free(port);
1839         }
1840
1841         /* Output port types. */
1842         for ( ; ; ) {
1843                 struct port_out_type *elem;
1844
1845                 elem = TAILQ_FIRST(&p->port_out_types);
1846                 if (!elem)
1847                         break;
1848
1849                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1850                 free(elem);
1851         }
1852 }
1853
1854 /*
1855  * Extern object.
1856  */
1857 static struct extern_type *
1858 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1859 {
1860         struct extern_type *elem;
1861
1862         TAILQ_FOREACH(elem, &p->extern_types, node)
1863                 if (strcmp(elem->name, name) == 0)
1864                         return elem;
1865
1866         return NULL;
1867 }
1868
1869 static struct extern_type_member_func *
1870 extern_type_member_func_find(struct extern_type *type, const char *name)
1871 {
1872         struct extern_type_member_func *elem;
1873
1874         TAILQ_FOREACH(elem, &type->funcs, node)
1875                 if (strcmp(elem->name, name) == 0)
1876                         return elem;
1877
1878         return NULL;
1879 }
1880
1881 static struct extern_obj *
1882 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1883 {
1884         struct extern_obj *elem;
1885
1886         TAILQ_FOREACH(elem, &p->extern_objs, node)
1887                 if (strcmp(elem->name, name) == 0)
1888                         return elem;
1889
1890         return NULL;
1891 }
1892
1893 static struct extern_type_member_func *
1894 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1895                              const char *name,
1896                              struct extern_obj **obj)
1897 {
1898         struct extern_obj *object;
1899         struct extern_type_member_func *func;
1900         char *object_name, *func_name;
1901
1902         if (name[0] != 'e' || name[1] != '.')
1903                 return NULL;
1904
1905         object_name = strdup(&name[2]);
1906         if (!object_name)
1907                 return NULL;
1908
1909         func_name = strchr(object_name, '.');
1910         if (!func_name) {
1911                 free(object_name);
1912                 return NULL;
1913         }
1914
1915         *func_name = 0;
1916         func_name++;
1917
1918         object = extern_obj_find(p, object_name);
1919         if (!object) {
1920                 free(object_name);
1921                 return NULL;
1922         }
1923
1924         func = extern_type_member_func_find(object->type, func_name);
1925         if (!func) {
1926                 free(object_name);
1927                 return NULL;
1928         }
1929
1930         if (obj)
1931                 *obj = object;
1932
1933         free(object_name);
1934         return func;
1935 }
1936
1937 static struct field *
1938 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1939                                const char *name,
1940                                struct extern_obj **object)
1941 {
1942         struct extern_obj *obj;
1943         struct field *f;
1944         char *obj_name, *field_name;
1945
1946         if ((name[0] != 'e') || (name[1] != '.'))
1947                 return NULL;
1948
1949         obj_name = strdup(&name[2]);
1950         if (!obj_name)
1951                 return NULL;
1952
1953         field_name = strchr(obj_name, '.');
1954         if (!field_name) {
1955                 free(obj_name);
1956                 return NULL;
1957         }
1958
1959         *field_name = 0;
1960         field_name++;
1961
1962         obj = extern_obj_find(p, obj_name);
1963         if (!obj) {
1964                 free(obj_name);
1965                 return NULL;
1966         }
1967
1968         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1969         if (!f) {
1970                 free(obj_name);
1971                 return NULL;
1972         }
1973
1974         if (object)
1975                 *object = obj;
1976
1977         free(obj_name);
1978         return f;
1979 }
1980
1981 int
1982 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1983         const char *name,
1984         const char *mailbox_struct_type_name,
1985         rte_swx_extern_type_constructor_t constructor,
1986         rte_swx_extern_type_destructor_t destructor)
1987 {
1988         struct extern_type *elem;
1989         struct struct_type *mailbox_struct_type;
1990
1991         CHECK(p, EINVAL);
1992
1993         CHECK_NAME(name, EINVAL);
1994         CHECK(!extern_type_find(p, name), EEXIST);
1995
1996         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1997         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1998         CHECK(mailbox_struct_type, EINVAL);
1999         CHECK(!mailbox_struct_type->var_size, EINVAL);
2000
2001         CHECK(constructor, EINVAL);
2002         CHECK(destructor, EINVAL);
2003
2004         /* Node allocation. */
2005         elem = calloc(1, sizeof(struct extern_type));
2006         CHECK(elem, ENOMEM);
2007
2008         /* Node initialization. */
2009         strcpy(elem->name, name);
2010         elem->mailbox_struct_type = mailbox_struct_type;
2011         elem->constructor = constructor;
2012         elem->destructor = destructor;
2013         TAILQ_INIT(&elem->funcs);
2014
2015         /* Node add to tailq. */
2016         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
2017
2018         return 0;
2019 }
2020
2021 int
2022 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
2023         const char *extern_type_name,
2024         const char *name,
2025         rte_swx_extern_type_member_func_t member_func)
2026 {
2027         struct extern_type *type;
2028         struct extern_type_member_func *type_member;
2029
2030         CHECK(p, EINVAL);
2031
2032         CHECK_NAME(extern_type_name, EINVAL);
2033         type = extern_type_find(p, extern_type_name);
2034         CHECK(type, EINVAL);
2035         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
2036
2037         CHECK_NAME(name, EINVAL);
2038         CHECK(!extern_type_member_func_find(type, name), EEXIST);
2039
2040         CHECK(member_func, EINVAL);
2041
2042         /* Node allocation. */
2043         type_member = calloc(1, sizeof(struct extern_type_member_func));
2044         CHECK(type_member, ENOMEM);
2045
2046         /* Node initialization. */
2047         strcpy(type_member->name, name);
2048         type_member->func = member_func;
2049         type_member->id = type->n_funcs;
2050
2051         /* Node add to tailq. */
2052         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
2053         type->n_funcs++;
2054
2055         return 0;
2056 }
2057
2058 int
2059 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
2060                                       const char *extern_type_name,
2061                                       const char *name,
2062                                       const char *args)
2063 {
2064         struct extern_type *type;
2065         struct extern_obj *obj;
2066         void *obj_handle;
2067
2068         CHECK(p, EINVAL);
2069
2070         CHECK_NAME(extern_type_name, EINVAL);
2071         type = extern_type_find(p, extern_type_name);
2072         CHECK(type, EINVAL);
2073
2074         CHECK_NAME(name, EINVAL);
2075         CHECK(!extern_obj_find(p, name), EEXIST);
2076
2077         /* Node allocation. */
2078         obj = calloc(1, sizeof(struct extern_obj));
2079         CHECK(obj, ENOMEM);
2080
2081         /* Object construction. */
2082         obj_handle = type->constructor(args);
2083         if (!obj_handle) {
2084                 free(obj);
2085                 CHECK(0, ENODEV);
2086         }
2087
2088         /* Node initialization. */
2089         strcpy(obj->name, name);
2090         obj->type = type;
2091         obj->obj = obj_handle;
2092         obj->struct_id = p->n_structs;
2093         obj->id = p->n_extern_objs;
2094
2095         /* Node add to tailq. */
2096         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
2097         p->n_extern_objs++;
2098         p->n_structs++;
2099
2100         return 0;
2101 }
2102
2103 static int
2104 extern_obj_build(struct rte_swx_pipeline *p)
2105 {
2106         uint32_t i;
2107
2108         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2109                 struct thread *t = &p->threads[i];
2110                 struct extern_obj *obj;
2111
2112                 t->extern_objs = calloc(p->n_extern_objs,
2113                                         sizeof(struct extern_obj_runtime));
2114                 CHECK(t->extern_objs, ENOMEM);
2115
2116                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
2117                         struct extern_obj_runtime *r =
2118                                 &t->extern_objs[obj->id];
2119                         struct extern_type_member_func *func;
2120                         uint32_t mailbox_size =
2121                                 obj->type->mailbox_struct_type->n_bits / 8;
2122
2123                         r->obj = obj->obj;
2124
2125                         r->mailbox = calloc(1, mailbox_size);
2126                         CHECK(r->mailbox, ENOMEM);
2127
2128                         TAILQ_FOREACH(func, &obj->type->funcs, node)
2129                                 r->funcs[func->id] = func->func;
2130
2131                         t->structs[obj->struct_id] = r->mailbox;
2132                 }
2133         }
2134
2135         return 0;
2136 }
2137
2138 static void
2139 extern_obj_build_free(struct rte_swx_pipeline *p)
2140 {
2141         uint32_t i;
2142
2143         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2144                 struct thread *t = &p->threads[i];
2145                 uint32_t j;
2146
2147                 if (!t->extern_objs)
2148                         continue;
2149
2150                 for (j = 0; j < p->n_extern_objs; j++) {
2151                         struct extern_obj_runtime *r = &t->extern_objs[j];
2152
2153                         free(r->mailbox);
2154                 }
2155
2156                 free(t->extern_objs);
2157                 t->extern_objs = NULL;
2158         }
2159 }
2160
2161 static void
2162 extern_obj_free(struct rte_swx_pipeline *p)
2163 {
2164         extern_obj_build_free(p);
2165
2166         /* Extern objects. */
2167         for ( ; ; ) {
2168                 struct extern_obj *elem;
2169
2170                 elem = TAILQ_FIRST(&p->extern_objs);
2171                 if (!elem)
2172                         break;
2173
2174                 TAILQ_REMOVE(&p->extern_objs, elem, node);
2175                 if (elem->obj)
2176                         elem->type->destructor(elem->obj);
2177                 free(elem);
2178         }
2179
2180         /* Extern types. */
2181         for ( ; ; ) {
2182                 struct extern_type *elem;
2183
2184                 elem = TAILQ_FIRST(&p->extern_types);
2185                 if (!elem)
2186                         break;
2187
2188                 TAILQ_REMOVE(&p->extern_types, elem, node);
2189
2190                 for ( ; ; ) {
2191                         struct extern_type_member_func *func;
2192
2193                         func = TAILQ_FIRST(&elem->funcs);
2194                         if (!func)
2195                                 break;
2196
2197                         TAILQ_REMOVE(&elem->funcs, func, node);
2198                         free(func);
2199                 }
2200
2201                 free(elem);
2202         }
2203 }
2204
2205 /*
2206  * Extern function.
2207  */
2208 static struct extern_func *
2209 extern_func_find(struct rte_swx_pipeline *p, const char *name)
2210 {
2211         struct extern_func *elem;
2212
2213         TAILQ_FOREACH(elem, &p->extern_funcs, node)
2214                 if (strcmp(elem->name, name) == 0)
2215                         return elem;
2216
2217         return NULL;
2218 }
2219
2220 static struct extern_func *
2221 extern_func_parse(struct rte_swx_pipeline *p,
2222                   const char *name)
2223 {
2224         if (name[0] != 'f' || name[1] != '.')
2225                 return NULL;
2226
2227         return extern_func_find(p, &name[2]);
2228 }
2229
2230 static struct field *
2231 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
2232                                 const char *name,
2233                                 struct extern_func **function)
2234 {
2235         struct extern_func *func;
2236         struct field *f;
2237         char *func_name, *field_name;
2238
2239         if ((name[0] != 'f') || (name[1] != '.'))
2240                 return NULL;
2241
2242         func_name = strdup(&name[2]);
2243         if (!func_name)
2244                 return NULL;
2245
2246         field_name = strchr(func_name, '.');
2247         if (!field_name) {
2248                 free(func_name);
2249                 return NULL;
2250         }
2251
2252         *field_name = 0;
2253         field_name++;
2254
2255         func = extern_func_find(p, func_name);
2256         if (!func) {
2257                 free(func_name);
2258                 return NULL;
2259         }
2260
2261         f = struct_type_field_find(func->mailbox_struct_type, field_name);
2262         if (!f) {
2263                 free(func_name);
2264                 return NULL;
2265         }
2266
2267         if (function)
2268                 *function = func;
2269
2270         free(func_name);
2271         return f;
2272 }
2273
2274 int
2275 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
2276                                       const char *name,
2277                                       const char *mailbox_struct_type_name,
2278                                       rte_swx_extern_func_t func)
2279 {
2280         struct extern_func *f;
2281         struct struct_type *mailbox_struct_type;
2282
2283         CHECK(p, EINVAL);
2284
2285         CHECK_NAME(name, EINVAL);
2286         CHECK(!extern_func_find(p, name), EEXIST);
2287
2288         CHECK_NAME(mailbox_struct_type_name, EINVAL);
2289         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2290         CHECK(mailbox_struct_type, EINVAL);
2291         CHECK(!mailbox_struct_type->var_size, EINVAL);
2292
2293         CHECK(func, EINVAL);
2294
2295         /* Node allocation. */
2296         f = calloc(1, sizeof(struct extern_func));
2297         CHECK(func, ENOMEM);
2298
2299         /* Node initialization. */
2300         strcpy(f->name, name);
2301         f->mailbox_struct_type = mailbox_struct_type;
2302         f->func = func;
2303         f->struct_id = p->n_structs;
2304         f->id = p->n_extern_funcs;
2305
2306         /* Node add to tailq. */
2307         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
2308         p->n_extern_funcs++;
2309         p->n_structs++;
2310
2311         return 0;
2312 }
2313
2314 static int
2315 extern_func_build(struct rte_swx_pipeline *p)
2316 {
2317         uint32_t i;
2318
2319         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2320                 struct thread *t = &p->threads[i];
2321                 struct extern_func *func;
2322
2323                 /* Memory allocation. */
2324                 t->extern_funcs = calloc(p->n_extern_funcs,
2325                                          sizeof(struct extern_func_runtime));
2326                 CHECK(t->extern_funcs, ENOMEM);
2327
2328                 /* Extern function. */
2329                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
2330                         struct extern_func_runtime *r =
2331                                 &t->extern_funcs[func->id];
2332                         uint32_t mailbox_size =
2333                                 func->mailbox_struct_type->n_bits / 8;
2334
2335                         r->func = func->func;
2336
2337                         r->mailbox = calloc(1, mailbox_size);
2338                         CHECK(r->mailbox, ENOMEM);
2339
2340                         t->structs[func->struct_id] = r->mailbox;
2341                 }
2342         }
2343
2344         return 0;
2345 }
2346
2347 static void
2348 extern_func_build_free(struct rte_swx_pipeline *p)
2349 {
2350         uint32_t i;
2351
2352         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2353                 struct thread *t = &p->threads[i];
2354                 uint32_t j;
2355
2356                 if (!t->extern_funcs)
2357                         continue;
2358
2359                 for (j = 0; j < p->n_extern_funcs; j++) {
2360                         struct extern_func_runtime *r = &t->extern_funcs[j];
2361
2362                         free(r->mailbox);
2363                 }
2364
2365                 free(t->extern_funcs);
2366                 t->extern_funcs = NULL;
2367         }
2368 }
2369
2370 static void
2371 extern_func_free(struct rte_swx_pipeline *p)
2372 {
2373         extern_func_build_free(p);
2374
2375         for ( ; ; ) {
2376                 struct extern_func *elem;
2377
2378                 elem = TAILQ_FIRST(&p->extern_funcs);
2379                 if (!elem)
2380                         break;
2381
2382                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
2383                 free(elem);
2384         }
2385 }
2386
2387 /*
2388  * Header.
2389  */
2390 static struct header *
2391 header_find(struct rte_swx_pipeline *p, const char *name)
2392 {
2393         struct header *elem;
2394
2395         TAILQ_FOREACH(elem, &p->headers, node)
2396                 if (strcmp(elem->name, name) == 0)
2397                         return elem;
2398
2399         return NULL;
2400 }
2401
2402 static struct header *
2403 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
2404 {
2405         struct header *elem;
2406
2407         TAILQ_FOREACH(elem, &p->headers, node)
2408                 if (elem->struct_id == struct_id)
2409                         return elem;
2410
2411         return NULL;
2412 }
2413
2414 static struct header *
2415 header_parse(struct rte_swx_pipeline *p,
2416              const char *name)
2417 {
2418         if (name[0] != 'h' || name[1] != '.')
2419                 return NULL;
2420
2421         return header_find(p, &name[2]);
2422 }
2423
2424 static struct field *
2425 header_field_parse(struct rte_swx_pipeline *p,
2426                    const char *name,
2427                    struct header **header)
2428 {
2429         struct header *h;
2430         struct field *f;
2431         char *header_name, *field_name;
2432
2433         if ((name[0] != 'h') || (name[1] != '.'))
2434                 return NULL;
2435
2436         header_name = strdup(&name[2]);
2437         if (!header_name)
2438                 return NULL;
2439
2440         field_name = strchr(header_name, '.');
2441         if (!field_name) {
2442                 free(header_name);
2443                 return NULL;
2444         }
2445
2446         *field_name = 0;
2447         field_name++;
2448
2449         h = header_find(p, header_name);
2450         if (!h) {
2451                 free(header_name);
2452                 return NULL;
2453         }
2454
2455         f = struct_type_field_find(h->st, field_name);
2456         if (!f) {
2457                 free(header_name);
2458                 return NULL;
2459         }
2460
2461         if (header)
2462                 *header = h;
2463
2464         free(header_name);
2465         return f;
2466 }
2467
2468 int
2469 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2470                                         const char *name,
2471                                         const char *struct_type_name)
2472 {
2473         struct struct_type *st;
2474         struct header *h;
2475         size_t n_headers_max;
2476
2477         CHECK(p, EINVAL);
2478         CHECK_NAME(name, EINVAL);
2479         CHECK_NAME(struct_type_name, EINVAL);
2480
2481         CHECK(!header_find(p, name), EEXIST);
2482
2483         st = struct_type_find(p, struct_type_name);
2484         CHECK(st, EINVAL);
2485
2486         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2487         CHECK(p->n_headers < n_headers_max, ENOSPC);
2488
2489         /* Node allocation. */
2490         h = calloc(1, sizeof(struct header));
2491         CHECK(h, ENOMEM);
2492
2493         /* Node initialization. */
2494         strcpy(h->name, name);
2495         h->st = st;
2496         h->struct_id = p->n_structs;
2497         h->id = p->n_headers;
2498
2499         /* Node add to tailq. */
2500         TAILQ_INSERT_TAIL(&p->headers, h, node);
2501         p->n_headers++;
2502         p->n_structs++;
2503
2504         return 0;
2505 }
2506
2507 static int
2508 header_build(struct rte_swx_pipeline *p)
2509 {
2510         struct header *h;
2511         uint32_t n_bytes = 0, i;
2512
2513         TAILQ_FOREACH(h, &p->headers, node) {
2514                 n_bytes += h->st->n_bits / 8;
2515         }
2516
2517         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2518                 struct thread *t = &p->threads[i];
2519                 uint32_t offset = 0;
2520
2521                 t->headers = calloc(p->n_headers,
2522                                     sizeof(struct header_runtime));
2523                 CHECK(t->headers, ENOMEM);
2524
2525                 t->headers_out = calloc(p->n_headers,
2526                                         sizeof(struct header_out_runtime));
2527                 CHECK(t->headers_out, ENOMEM);
2528
2529                 t->header_storage = calloc(1, n_bytes);
2530                 CHECK(t->header_storage, ENOMEM);
2531
2532                 t->header_out_storage = calloc(1, n_bytes);
2533                 CHECK(t->header_out_storage, ENOMEM);
2534
2535                 TAILQ_FOREACH(h, &p->headers, node) {
2536                         uint8_t *header_storage;
2537                         uint32_t n_bytes =  h->st->n_bits / 8;
2538
2539                         header_storage = &t->header_storage[offset];
2540                         offset += n_bytes;
2541
2542                         t->headers[h->id].ptr0 = header_storage;
2543                         t->headers[h->id].n_bytes = n_bytes;
2544
2545                         t->structs[h->struct_id] = header_storage;
2546                 }
2547         }
2548
2549         return 0;
2550 }
2551
2552 static void
2553 header_build_free(struct rte_swx_pipeline *p)
2554 {
2555         uint32_t i;
2556
2557         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2558                 struct thread *t = &p->threads[i];
2559
2560                 free(t->headers_out);
2561                 t->headers_out = NULL;
2562
2563                 free(t->headers);
2564                 t->headers = NULL;
2565
2566                 free(t->header_out_storage);
2567                 t->header_out_storage = NULL;
2568
2569                 free(t->header_storage);
2570                 t->header_storage = NULL;
2571         }
2572 }
2573
2574 static void
2575 header_free(struct rte_swx_pipeline *p)
2576 {
2577         header_build_free(p);
2578
2579         for ( ; ; ) {
2580                 struct header *elem;
2581
2582                 elem = TAILQ_FIRST(&p->headers);
2583                 if (!elem)
2584                         break;
2585
2586                 TAILQ_REMOVE(&p->headers, elem, node);
2587                 free(elem);
2588         }
2589 }
2590
2591 /*
2592  * Meta-data.
2593  */
2594 static struct field *
2595 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2596 {
2597         if (!p->metadata_st)
2598                 return NULL;
2599
2600         if (name[0] != 'm' || name[1] != '.')
2601                 return NULL;
2602
2603         return struct_type_field_find(p->metadata_st, &name[2]);
2604 }
2605
2606 int
2607 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2608                                           const char *struct_type_name)
2609 {
2610         struct struct_type *st = NULL;
2611
2612         CHECK(p, EINVAL);
2613
2614         CHECK_NAME(struct_type_name, EINVAL);
2615         st  = struct_type_find(p, struct_type_name);
2616         CHECK(st, EINVAL);
2617         CHECK(!st->var_size, EINVAL);
2618         CHECK(!p->metadata_st, EINVAL);
2619
2620         p->metadata_st = st;
2621         p->metadata_struct_id = p->n_structs;
2622
2623         p->n_structs++;
2624
2625         return 0;
2626 }
2627
2628 static int
2629 metadata_build(struct rte_swx_pipeline *p)
2630 {
2631         uint32_t n_bytes = p->metadata_st->n_bits / 8;
2632         uint32_t i;
2633
2634         /* Thread-level initialization. */
2635         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2636                 struct thread *t = &p->threads[i];
2637                 uint8_t *metadata;
2638
2639                 metadata = calloc(1, n_bytes);
2640                 CHECK(metadata, ENOMEM);
2641
2642                 t->metadata = metadata;
2643                 t->structs[p->metadata_struct_id] = metadata;
2644         }
2645
2646         return 0;
2647 }
2648
2649 static void
2650 metadata_build_free(struct rte_swx_pipeline *p)
2651 {
2652         uint32_t i;
2653
2654         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2655                 struct thread *t = &p->threads[i];
2656
2657                 free(t->metadata);
2658                 t->metadata = NULL;
2659         }
2660 }
2661
2662 static void
2663 metadata_free(struct rte_swx_pipeline *p)
2664 {
2665         metadata_build_free(p);
2666 }
2667
2668 /*
2669  * Instruction.
2670  */
2671 static int
2672 instruction_is_tx(enum instruction_type type)
2673 {
2674         switch (type) {
2675         case INSTR_TX:
2676         case INSTR_TX_I:
2677                 return 1;
2678
2679         default:
2680                 return 0;
2681         }
2682 }
2683
2684 static int
2685 instruction_is_jmp(struct instruction *instr)
2686 {
2687         switch (instr->type) {
2688         case INSTR_JMP:
2689         case INSTR_JMP_VALID:
2690         case INSTR_JMP_INVALID:
2691         case INSTR_JMP_HIT:
2692         case INSTR_JMP_MISS:
2693         case INSTR_JMP_ACTION_HIT:
2694         case INSTR_JMP_ACTION_MISS:
2695         case INSTR_JMP_EQ:
2696         case INSTR_JMP_EQ_MH:
2697         case INSTR_JMP_EQ_HM:
2698         case INSTR_JMP_EQ_HH:
2699         case INSTR_JMP_EQ_I:
2700         case INSTR_JMP_NEQ:
2701         case INSTR_JMP_NEQ_MH:
2702         case INSTR_JMP_NEQ_HM:
2703         case INSTR_JMP_NEQ_HH:
2704         case INSTR_JMP_NEQ_I:
2705         case INSTR_JMP_LT:
2706         case INSTR_JMP_LT_MH:
2707         case INSTR_JMP_LT_HM:
2708         case INSTR_JMP_LT_HH:
2709         case INSTR_JMP_LT_MI:
2710         case INSTR_JMP_LT_HI:
2711         case INSTR_JMP_GT:
2712         case INSTR_JMP_GT_MH:
2713         case INSTR_JMP_GT_HM:
2714         case INSTR_JMP_GT_HH:
2715         case INSTR_JMP_GT_MI:
2716         case INSTR_JMP_GT_HI:
2717                 return 1;
2718
2719         default:
2720                 return 0;
2721         }
2722 }
2723
2724 static struct field *
2725 action_field_parse(struct action *action, const char *name);
2726
2727 static struct field *
2728 struct_field_parse(struct rte_swx_pipeline *p,
2729                    struct action *action,
2730                    const char *name,
2731                    uint32_t *struct_id)
2732 {
2733         struct field *f;
2734
2735         switch (name[0]) {
2736         case 'h':
2737         {
2738                 struct header *header;
2739
2740                 f = header_field_parse(p, name, &header);
2741                 if (!f)
2742                         return NULL;
2743
2744                 *struct_id = header->struct_id;
2745                 return f;
2746         }
2747
2748         case 'm':
2749         {
2750                 f = metadata_field_parse(p, name);
2751                 if (!f)
2752                         return NULL;
2753
2754                 *struct_id = p->metadata_struct_id;
2755                 return f;
2756         }
2757
2758         case 't':
2759         {
2760                 if (!action)
2761                         return NULL;
2762
2763                 f = action_field_parse(action, name);
2764                 if (!f)
2765                         return NULL;
2766
2767                 *struct_id = 0;
2768                 return f;
2769         }
2770
2771         case 'e':
2772         {
2773                 struct extern_obj *obj;
2774
2775                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2776                 if (!f)
2777                         return NULL;
2778
2779                 *struct_id = obj->struct_id;
2780                 return f;
2781         }
2782
2783         case 'f':
2784         {
2785                 struct extern_func *func;
2786
2787                 f = extern_func_mailbox_field_parse(p, name, &func);
2788                 if (!f)
2789                         return NULL;
2790
2791                 *struct_id = func->struct_id;
2792                 return f;
2793         }
2794
2795         default:
2796                 return NULL;
2797         }
2798 }
2799
2800 static inline void
2801 pipeline_port_inc(struct rte_swx_pipeline *p)
2802 {
2803         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2804 }
2805
2806 static inline void
2807 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2808 {
2809         t->ip = p->instructions;
2810 }
2811
2812 static inline void
2813 thread_ip_set(struct thread *t, struct instruction *ip)
2814 {
2815         t->ip = ip;
2816 }
2817
2818 static inline void
2819 thread_ip_action_call(struct rte_swx_pipeline *p,
2820                       struct thread *t,
2821                       uint32_t action_id)
2822 {
2823         t->ret = t->ip + 1;
2824         t->ip = p->action_instructions[action_id];
2825 }
2826
2827 static inline void
2828 thread_ip_inc(struct rte_swx_pipeline *p);
2829
2830 static inline void
2831 thread_ip_inc(struct rte_swx_pipeline *p)
2832 {
2833         struct thread *t = &p->threads[p->thread_id];
2834
2835         t->ip++;
2836 }
2837
2838 static inline void
2839 thread_ip_inc_cond(struct thread *t, int cond)
2840 {
2841         t->ip += cond;
2842 }
2843
2844 static inline void
2845 thread_yield(struct rte_swx_pipeline *p)
2846 {
2847         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2848 }
2849
2850 static inline void
2851 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2852 {
2853         p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2854 }
2855
2856 /*
2857  * rx.
2858  */
2859 static int
2860 instr_rx_translate(struct rte_swx_pipeline *p,
2861                    struct action *action,
2862                    char **tokens,
2863                    int n_tokens,
2864                    struct instruction *instr,
2865                    struct instruction_data *data __rte_unused)
2866 {
2867         struct field *f;
2868
2869         CHECK(!action, EINVAL);
2870         CHECK(n_tokens == 2, EINVAL);
2871
2872         f = metadata_field_parse(p, tokens[1]);
2873         CHECK(f, EINVAL);
2874
2875         instr->type = INSTR_RX;
2876         instr->io.io.offset = f->offset / 8;
2877         instr->io.io.n_bits = f->n_bits;
2878         return 0;
2879 }
2880
2881 static inline void
2882 instr_rx_exec(struct rte_swx_pipeline *p);
2883
2884 static inline void
2885 instr_rx_exec(struct rte_swx_pipeline *p)
2886 {
2887         struct thread *t = &p->threads[p->thread_id];
2888         struct instruction *ip = t->ip;
2889         struct port_in_runtime *port = &p->in[p->port_id];
2890         struct rte_swx_pkt *pkt = &t->pkt;
2891         int pkt_received;
2892
2893         /* Packet. */
2894         pkt_received = port->pkt_rx(port->obj, pkt);
2895         t->ptr = &pkt->pkt[pkt->offset];
2896         rte_prefetch0(t->ptr);
2897
2898         TRACE("[Thread %2u] rx %s from port %u\n",
2899               p->thread_id,
2900               pkt_received ? "1 pkt" : "0 pkts",
2901               p->port_id);
2902
2903         /* Headers. */
2904         t->valid_headers = 0;
2905         t->n_headers_out = 0;
2906
2907         /* Meta-data. */
2908         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2909
2910         /* Tables. */
2911         t->table_state = p->table_state;
2912
2913         /* Thread. */
2914         pipeline_port_inc(p);
2915         thread_ip_inc_cond(t, pkt_received);
2916         thread_yield(p);
2917 }
2918
2919 /*
2920  * tx.
2921  */
2922 static int
2923 instr_tx_translate(struct rte_swx_pipeline *p,
2924                    struct action *action __rte_unused,
2925                    char **tokens,
2926                    int n_tokens,
2927                    struct instruction *instr,
2928                    struct instruction_data *data __rte_unused)
2929 {
2930         char *port = tokens[1];
2931         struct field *f;
2932         uint32_t port_val;
2933
2934         CHECK(n_tokens == 2, EINVAL);
2935
2936         f = metadata_field_parse(p, port);
2937         if (f) {
2938                 instr->type = INSTR_TX;
2939                 instr->io.io.offset = f->offset / 8;
2940                 instr->io.io.n_bits = f->n_bits;
2941                 return 0;
2942         }
2943
2944         /* TX_I. */
2945         port_val = strtoul(port, &port, 0);
2946         CHECK(!port[0], EINVAL);
2947
2948         instr->type = INSTR_TX_I;
2949         instr->io.io.val = port_val;
2950         return 0;
2951 }
2952
2953 static int
2954 instr_drop_translate(struct rte_swx_pipeline *p,
2955                      struct action *action __rte_unused,
2956                      char **tokens __rte_unused,
2957                      int n_tokens,
2958                      struct instruction *instr,
2959                      struct instruction_data *data __rte_unused)
2960 {
2961         CHECK(n_tokens == 1, EINVAL);
2962
2963         /* TX_I. */
2964         instr->type = INSTR_TX_I;
2965         instr->io.io.val = p->n_ports_out - 1;
2966         return 0;
2967 }
2968
2969 static inline void
2970 emit_handler(struct thread *t)
2971 {
2972         struct header_out_runtime *h0 = &t->headers_out[0];
2973         struct header_out_runtime *h1 = &t->headers_out[1];
2974         uint32_t offset = 0, i;
2975
2976         /* No header change or header decapsulation. */
2977         if ((t->n_headers_out == 1) &&
2978             (h0->ptr + h0->n_bytes == t->ptr)) {
2979                 TRACE("Emit handler: no header change or header decap.\n");
2980
2981                 t->pkt.offset -= h0->n_bytes;
2982                 t->pkt.length += h0->n_bytes;
2983
2984                 return;
2985         }
2986
2987         /* Header encapsulation (optionally, with prior header decasulation). */
2988         if ((t->n_headers_out == 2) &&
2989             (h1->ptr + h1->n_bytes == t->ptr) &&
2990             (h0->ptr == h0->ptr0)) {
2991                 uint32_t offset;
2992
2993                 TRACE("Emit handler: header encapsulation.\n");
2994
2995                 offset = h0->n_bytes + h1->n_bytes;
2996                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2997                 t->pkt.offset -= offset;
2998                 t->pkt.length += offset;
2999
3000                 return;
3001         }
3002
3003         /* Header insertion. */
3004         /* TBD */
3005
3006         /* Header extraction. */
3007         /* TBD */
3008
3009         /* For any other case. */
3010         TRACE("Emit handler: complex case.\n");
3011
3012         for (i = 0; i < t->n_headers_out; i++) {
3013                 struct header_out_runtime *h = &t->headers_out[i];
3014
3015                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
3016                 offset += h->n_bytes;
3017         }
3018
3019         if (offset) {
3020                 memcpy(t->ptr - offset, t->header_out_storage, offset);
3021                 t->pkt.offset -= offset;
3022                 t->pkt.length += offset;
3023         }
3024 }
3025
3026 static inline void
3027 instr_tx_exec(struct rte_swx_pipeline *p);
3028
3029 static inline void
3030 instr_tx_exec(struct rte_swx_pipeline *p)
3031 {
3032         struct thread *t = &p->threads[p->thread_id];
3033         struct instruction *ip = t->ip;
3034         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
3035         struct port_out_runtime *port = &p->out[port_id];
3036         struct rte_swx_pkt *pkt = &t->pkt;
3037
3038         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
3039               p->thread_id,
3040               (uint32_t)port_id);
3041
3042         /* Headers. */
3043         emit_handler(t);
3044
3045         /* Packet. */
3046         port->pkt_tx(port->obj, pkt);
3047
3048         /* Thread. */
3049         thread_ip_reset(p, t);
3050         instr_rx_exec(p);
3051 }
3052
3053 static inline void
3054 instr_tx_i_exec(struct rte_swx_pipeline *p)
3055 {
3056         struct thread *t = &p->threads[p->thread_id];
3057         struct instruction *ip = t->ip;
3058         uint64_t port_id = ip->io.io.val;
3059         struct port_out_runtime *port = &p->out[port_id];
3060         struct rte_swx_pkt *pkt = &t->pkt;
3061
3062         TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
3063               p->thread_id,
3064               (uint32_t)port_id);
3065
3066         /* Headers. */
3067         emit_handler(t);
3068
3069         /* Packet. */
3070         port->pkt_tx(port->obj, pkt);
3071
3072         /* Thread. */
3073         thread_ip_reset(p, t);
3074         instr_rx_exec(p);
3075 }
3076
3077 /*
3078  * extract.
3079  */
3080 static int
3081 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
3082                             struct action *action,
3083                             char **tokens,
3084                             int n_tokens,
3085                             struct instruction *instr,
3086                             struct instruction_data *data __rte_unused)
3087 {
3088         struct header *h;
3089
3090         CHECK(!action, EINVAL);
3091         CHECK(n_tokens == 2, EINVAL);
3092
3093         h = header_parse(p, tokens[1]);
3094         CHECK(h, EINVAL);
3095         CHECK(!h->st->var_size, EINVAL);
3096
3097         instr->type = INSTR_HDR_EXTRACT;
3098         instr->io.hdr.header_id[0] = h->id;
3099         instr->io.hdr.struct_id[0] = h->struct_id;
3100         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3101         return 0;
3102 }
3103
3104 static inline void
3105 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
3106
3107 static inline void
3108 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
3109 {
3110         struct thread *t = &p->threads[p->thread_id];
3111         struct instruction *ip = t->ip;
3112         uint64_t valid_headers = t->valid_headers;
3113         uint8_t *ptr = t->ptr;
3114         uint32_t offset = t->pkt.offset;
3115         uint32_t length = t->pkt.length;
3116         uint32_t i;
3117
3118         for (i = 0; i < n_extract; i++) {
3119                 uint32_t header_id = ip->io.hdr.header_id[i];
3120                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3121                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3122
3123                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
3124                       p->thread_id,
3125                       header_id,
3126                       n_bytes);
3127
3128                 /* Headers. */
3129                 t->structs[struct_id] = ptr;
3130                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3131
3132                 /* Packet. */
3133                 offset += n_bytes;
3134                 length -= n_bytes;
3135                 ptr += n_bytes;
3136         }
3137
3138         /* Headers. */
3139         t->valid_headers = valid_headers;
3140
3141         /* Packet. */
3142         t->pkt.offset = offset;
3143         t->pkt.length = length;
3144         t->ptr = ptr;
3145 }
3146
3147 static inline void
3148 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
3149 {
3150         __instr_hdr_extract_exec(p, 1);
3151
3152         /* Thread. */
3153         thread_ip_inc(p);
3154 }
3155
3156 static inline void
3157 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
3158 {
3159         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3160               p->thread_id);
3161
3162         __instr_hdr_extract_exec(p, 2);
3163
3164         /* Thread. */
3165         thread_ip_inc(p);
3166 }
3167
3168 static inline void
3169 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
3170 {
3171         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3172               p->thread_id);
3173
3174         __instr_hdr_extract_exec(p, 3);
3175
3176         /* Thread. */
3177         thread_ip_inc(p);
3178 }
3179
3180 static inline void
3181 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
3182 {
3183         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3184               p->thread_id);
3185
3186         __instr_hdr_extract_exec(p, 4);
3187
3188         /* Thread. */
3189         thread_ip_inc(p);
3190 }
3191
3192 static inline void
3193 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
3194 {
3195         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3196               p->thread_id);
3197
3198         __instr_hdr_extract_exec(p, 5);
3199
3200         /* Thread. */
3201         thread_ip_inc(p);
3202 }
3203
3204 static inline void
3205 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
3206 {
3207         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3208               p->thread_id);
3209
3210         __instr_hdr_extract_exec(p, 6);
3211
3212         /* Thread. */
3213         thread_ip_inc(p);
3214 }
3215
3216 static inline void
3217 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
3218 {
3219         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3220               p->thread_id);
3221
3222         __instr_hdr_extract_exec(p, 7);
3223
3224         /* Thread. */
3225         thread_ip_inc(p);
3226 }
3227
3228 static inline void
3229 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
3230 {
3231         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3232               p->thread_id);
3233
3234         __instr_hdr_extract_exec(p, 8);
3235
3236         /* Thread. */
3237         thread_ip_inc(p);
3238 }
3239
3240 /*
3241  * emit.
3242  */
3243 static int
3244 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
3245                          struct action *action __rte_unused,
3246                          char **tokens,
3247                          int n_tokens,
3248                          struct instruction *instr,
3249                          struct instruction_data *data __rte_unused)
3250 {
3251         struct header *h;
3252
3253         CHECK(n_tokens == 2, EINVAL);
3254
3255         h = header_parse(p, tokens[1]);
3256         CHECK(h, EINVAL);
3257
3258         instr->type = INSTR_HDR_EMIT;
3259         instr->io.hdr.header_id[0] = h->id;
3260         instr->io.hdr.struct_id[0] = h->struct_id;
3261         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3262         return 0;
3263 }
3264
3265 static inline void
3266 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
3267
3268 static inline void
3269 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
3270 {
3271         struct thread *t = &p->threads[p->thread_id];
3272         struct instruction *ip = t->ip;
3273         uint64_t valid_headers = t->valid_headers;
3274         uint32_t n_headers_out = t->n_headers_out;
3275         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
3276         uint8_t *ho_ptr = NULL;
3277         uint32_t ho_nbytes = 0, first = 1, i;
3278
3279         for (i = 0; i < n_emit; i++) {
3280                 uint32_t header_id = ip->io.hdr.header_id[i];
3281                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3282
3283                 struct header_runtime *hi = &t->headers[header_id];
3284                 uint8_t *hi_ptr0 = hi->ptr0;
3285                 uint32_t n_bytes = hi->n_bytes;
3286
3287                 uint8_t *hi_ptr = t->structs[struct_id];
3288
3289                 if (!MASK64_BIT_GET(valid_headers, header_id))
3290                         continue;
3291
3292                 TRACE("[Thread %2u]: emit header %u\n",
3293                       p->thread_id,
3294                       header_id);
3295
3296                 /* Headers. */
3297                 if (first) {
3298                         first = 0;
3299
3300                         if (!t->n_headers_out) {
3301                                 ho = &t->headers_out[0];
3302
3303                                 ho->ptr0 = hi_ptr0;
3304                                 ho->ptr = hi_ptr;
3305
3306                                 ho_ptr = hi_ptr;
3307                                 ho_nbytes = n_bytes;
3308
3309                                 n_headers_out = 1;
3310
3311                                 continue;
3312                         } else {
3313                                 ho_ptr = ho->ptr;
3314                                 ho_nbytes = ho->n_bytes;
3315                         }
3316                 }
3317
3318                 if (ho_ptr + ho_nbytes == hi_ptr) {
3319                         ho_nbytes += n_bytes;
3320                 } else {
3321                         ho->n_bytes = ho_nbytes;
3322
3323                         ho++;
3324                         ho->ptr0 = hi_ptr0;
3325                         ho->ptr = hi_ptr;
3326
3327                         ho_ptr = hi_ptr;
3328                         ho_nbytes = n_bytes;
3329
3330                         n_headers_out++;
3331                 }
3332         }
3333
3334         ho->n_bytes = ho_nbytes;
3335         t->n_headers_out = n_headers_out;
3336 }
3337
3338 static inline void
3339 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
3340 {
3341         __instr_hdr_emit_exec(p, 1);
3342
3343         /* Thread. */
3344         thread_ip_inc(p);
3345 }
3346
3347 static inline void
3348 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
3349 {
3350         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3351               p->thread_id);
3352
3353         __instr_hdr_emit_exec(p, 1);
3354         instr_tx_exec(p);
3355 }
3356
3357 static inline void
3358 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
3359 {
3360         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3361               p->thread_id);
3362
3363         __instr_hdr_emit_exec(p, 2);
3364         instr_tx_exec(p);
3365 }
3366
3367 static inline void
3368 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
3369 {
3370         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3371               p->thread_id);
3372
3373         __instr_hdr_emit_exec(p, 3);
3374         instr_tx_exec(p);
3375 }
3376
3377 static inline void
3378 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
3379 {
3380         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3381               p->thread_id);
3382
3383         __instr_hdr_emit_exec(p, 4);
3384         instr_tx_exec(p);
3385 }
3386
3387 static inline void
3388 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
3389 {
3390         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3391               p->thread_id);
3392
3393         __instr_hdr_emit_exec(p, 5);
3394         instr_tx_exec(p);
3395 }
3396
3397 static inline void
3398 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
3399 {
3400         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3401               p->thread_id);
3402
3403         __instr_hdr_emit_exec(p, 6);
3404         instr_tx_exec(p);
3405 }
3406
3407 static inline void
3408 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
3409 {
3410         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3411               p->thread_id);
3412
3413         __instr_hdr_emit_exec(p, 7);
3414         instr_tx_exec(p);
3415 }
3416
3417 static inline void
3418 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
3419 {
3420         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
3421               p->thread_id);
3422
3423         __instr_hdr_emit_exec(p, 8);
3424         instr_tx_exec(p);
3425 }
3426
3427 /*
3428  * validate.
3429  */
3430 static int
3431 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
3432                              struct action *action __rte_unused,
3433                              char **tokens,
3434                              int n_tokens,
3435                              struct instruction *instr,
3436                              struct instruction_data *data __rte_unused)
3437 {
3438         struct header *h;
3439
3440         CHECK(n_tokens == 2, EINVAL);
3441
3442         h = header_parse(p, tokens[1]);
3443         CHECK(h, EINVAL);
3444
3445         instr->type = INSTR_HDR_VALIDATE;
3446         instr->valid.header_id = h->id;
3447         return 0;
3448 }
3449
3450 static inline void
3451 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
3452 {
3453         struct thread *t = &p->threads[p->thread_id];
3454         struct instruction *ip = t->ip;
3455         uint32_t header_id = ip->valid.header_id;
3456
3457         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
3458
3459         /* Headers. */
3460         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
3461
3462         /* Thread. */
3463         thread_ip_inc(p);
3464 }
3465
3466 /*
3467  * invalidate.
3468  */
3469 static int
3470 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
3471                                struct action *action __rte_unused,
3472                                char **tokens,
3473                                int n_tokens,
3474                                struct instruction *instr,
3475                                struct instruction_data *data __rte_unused)
3476 {
3477         struct header *h;
3478
3479         CHECK(n_tokens == 2, EINVAL);
3480
3481         h = header_parse(p, tokens[1]);
3482         CHECK(h, EINVAL);
3483
3484         instr->type = INSTR_HDR_INVALIDATE;
3485         instr->valid.header_id = h->id;
3486         return 0;
3487 }
3488
3489 static inline void
3490 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3491 {
3492         struct thread *t = &p->threads[p->thread_id];
3493         struct instruction *ip = t->ip;
3494         uint32_t header_id = ip->valid.header_id;
3495
3496         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3497
3498         /* Headers. */
3499         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3500
3501         /* Thread. */
3502         thread_ip_inc(p);
3503 }
3504
3505 /*
3506  * table.
3507  */
3508 static struct table *
3509 table_find(struct rte_swx_pipeline *p, const char *name);
3510
3511 static struct selector *
3512 selector_find(struct rte_swx_pipeline *p, const char *name);
3513
3514 static int
3515 instr_table_translate(struct rte_swx_pipeline *p,
3516                       struct action *action,
3517                       char **tokens,
3518                       int n_tokens,
3519                       struct instruction *instr,
3520                       struct instruction_data *data __rte_unused)
3521 {
3522         struct table *t;
3523         struct selector *s;
3524
3525         CHECK(!action, EINVAL);
3526         CHECK(n_tokens == 2, EINVAL);
3527
3528         t = table_find(p, tokens[1]);
3529         if (t) {
3530                 instr->type = INSTR_TABLE;
3531                 instr->table.table_id = t->id;
3532                 return 0;
3533         }
3534
3535         s = selector_find(p, tokens[1]);
3536         if (s) {
3537                 instr->type = INSTR_SELECTOR;
3538                 instr->table.table_id = s->id;
3539                 return 0;
3540         }
3541
3542         CHECK(0, EINVAL);
3543 }
3544
3545 static inline void
3546 instr_table_exec(struct rte_swx_pipeline *p)
3547 {
3548         struct thread *t = &p->threads[p->thread_id];
3549         struct instruction *ip = t->ip;
3550         uint32_t table_id = ip->table.table_id;
3551         struct rte_swx_table_state *ts = &t->table_state[table_id];
3552         struct table_runtime *table = &t->tables[table_id];
3553         struct table_statistics *stats = &p->table_stats[table_id];
3554         uint64_t action_id, n_pkts_hit, n_pkts_action;
3555         uint8_t *action_data;
3556         int done, hit;
3557
3558         /* Table. */
3559         done = table->func(ts->obj,
3560                            table->mailbox,
3561                            table->key,
3562                            &action_id,
3563                            &action_data,
3564                            &hit);
3565         if (!done) {
3566                 /* Thread. */
3567                 TRACE("[Thread %2u] table %u (not finalized)\n",
3568                       p->thread_id,
3569                       table_id);
3570
3571                 thread_yield(p);
3572                 return;
3573         }
3574
3575         action_id = hit ? action_id : ts->default_action_id;
3576         action_data = hit ? action_data : ts->default_action_data;
3577         n_pkts_hit = stats->n_pkts_hit[hit];
3578         n_pkts_action = stats->n_pkts_action[action_id];
3579
3580         TRACE("[Thread %2u] table %u (%s, action %u)\n",
3581               p->thread_id,
3582               table_id,
3583               hit ? "hit" : "miss",
3584               (uint32_t)action_id);
3585
3586         t->action_id = action_id;
3587         t->structs[0] = action_data;
3588         t->hit = hit;
3589         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
3590         stats->n_pkts_action[action_id] = n_pkts_action + 1;
3591
3592         /* Thread. */
3593         thread_ip_action_call(p, t, action_id);
3594 }
3595
3596 static inline void
3597 instr_selector_exec(struct rte_swx_pipeline *p)
3598 {
3599         struct thread *t = &p->threads[p->thread_id];
3600         struct instruction *ip = t->ip;
3601         uint32_t selector_id = ip->table.table_id;
3602         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
3603         struct selector_runtime *selector = &t->selectors[selector_id];
3604         struct selector_statistics *stats = &p->selector_stats[selector_id];
3605         uint64_t n_pkts = stats->n_pkts;
3606         int done;
3607
3608         /* Table. */
3609         done = rte_swx_table_selector_select(ts->obj,
3610                            selector->mailbox,
3611                            selector->group_id_buffer,
3612                            selector->selector_buffer,
3613                            selector->member_id_buffer);
3614         if (!done) {
3615                 /* Thread. */
3616                 TRACE("[Thread %2u] selector %u (not finalized)\n",
3617                       p->thread_id,
3618                       selector_id);
3619
3620                 thread_yield(p);
3621                 return;
3622         }
3623
3624
3625         TRACE("[Thread %2u] selector %u\n",
3626               p->thread_id,
3627               selector_id);
3628
3629         stats->n_pkts = n_pkts + 1;
3630
3631         /* Thread. */
3632         thread_ip_inc(p);
3633 }
3634
3635 /*
3636  * extern.
3637  */
3638 static int
3639 instr_extern_translate(struct rte_swx_pipeline *p,
3640                        struct action *action __rte_unused,
3641                        char **tokens,
3642                        int n_tokens,
3643                        struct instruction *instr,
3644                        struct instruction_data *data __rte_unused)
3645 {
3646         char *token = tokens[1];
3647
3648         CHECK(n_tokens == 2, EINVAL);
3649
3650         if (token[0] == 'e') {
3651                 struct extern_obj *obj;
3652                 struct extern_type_member_func *func;
3653
3654                 func = extern_obj_member_func_parse(p, token, &obj);
3655                 CHECK(func, EINVAL);
3656
3657                 instr->type = INSTR_EXTERN_OBJ;
3658                 instr->ext_obj.ext_obj_id = obj->id;
3659                 instr->ext_obj.func_id = func->id;
3660
3661                 return 0;
3662         }
3663
3664         if (token[0] == 'f') {
3665                 struct extern_func *func;
3666
3667                 func = extern_func_parse(p, token);
3668                 CHECK(func, EINVAL);
3669
3670                 instr->type = INSTR_EXTERN_FUNC;
3671                 instr->ext_func.ext_func_id = func->id;
3672
3673                 return 0;
3674         }
3675
3676         CHECK(0, EINVAL);
3677 }
3678
3679 static inline void
3680 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3681 {
3682         struct thread *t = &p->threads[p->thread_id];
3683         struct instruction *ip = t->ip;
3684         uint32_t obj_id = ip->ext_obj.ext_obj_id;
3685         uint32_t func_id = ip->ext_obj.func_id;
3686         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3687         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3688
3689         TRACE("[Thread %2u] extern obj %u member func %u\n",
3690               p->thread_id,
3691               obj_id,
3692               func_id);
3693
3694         /* Extern object member function execute. */
3695         uint32_t done = func(obj->obj, obj->mailbox);
3696
3697         /* Thread. */
3698         thread_ip_inc_cond(t, done);
3699         thread_yield_cond(p, done ^ 1);
3700 }
3701
3702 static inline void
3703 instr_extern_func_exec(struct rte_swx_pipeline *p)
3704 {
3705         struct thread *t = &p->threads[p->thread_id];
3706         struct instruction *ip = t->ip;
3707         uint32_t ext_func_id = ip->ext_func.ext_func_id;
3708         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3709         rte_swx_extern_func_t func = ext_func->func;
3710
3711         TRACE("[Thread %2u] extern func %u\n",
3712               p->thread_id,
3713               ext_func_id);
3714
3715         /* Extern function execute. */
3716         uint32_t done = func(ext_func->mailbox);
3717
3718         /* Thread. */
3719         thread_ip_inc_cond(t, done);
3720         thread_yield_cond(p, done ^ 1);
3721 }
3722
3723 /*
3724  * mov.
3725  */
3726 static int
3727 instr_mov_translate(struct rte_swx_pipeline *p,
3728                     struct action *action,
3729                     char **tokens,
3730                     int n_tokens,
3731                     struct instruction *instr,
3732                     struct instruction_data *data __rte_unused)
3733 {
3734         char *dst = tokens[1], *src = tokens[2];
3735         struct field *fdst, *fsrc;
3736         uint64_t src_val;
3737         uint32_t dst_struct_id = 0, src_struct_id = 0;
3738
3739         CHECK(n_tokens == 3, EINVAL);
3740
3741         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3742         CHECK(fdst, EINVAL);
3743         CHECK(!fdst->var_size, EINVAL);
3744
3745         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
3746         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3747         if (fsrc) {
3748                 CHECK(!fsrc->var_size, EINVAL);
3749
3750                 instr->type = INSTR_MOV;
3751                 if (dst[0] != 'h' && src[0] == 'h')
3752                         instr->type = INSTR_MOV_MH;
3753                 if (dst[0] == 'h' && src[0] != 'h')
3754                         instr->type = INSTR_MOV_HM;
3755                 if (dst[0] == 'h' && src[0] == 'h')
3756                         instr->type = INSTR_MOV_HH;
3757
3758                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3759                 instr->mov.dst.n_bits = fdst->n_bits;
3760                 instr->mov.dst.offset = fdst->offset / 8;
3761                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3762                 instr->mov.src.n_bits = fsrc->n_bits;
3763                 instr->mov.src.offset = fsrc->offset / 8;
3764                 return 0;
3765         }
3766
3767         /* MOV_I. */
3768         src_val = strtoull(src, &src, 0);
3769         CHECK(!src[0], EINVAL);
3770
3771         if (dst[0] == 'h')
3772                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3773
3774         instr->type = INSTR_MOV_I;
3775         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3776         instr->mov.dst.n_bits = fdst->n_bits;
3777         instr->mov.dst.offset = fdst->offset / 8;
3778         instr->mov.src_val = src_val;
3779         return 0;
3780 }
3781
3782 static inline void
3783 instr_mov_exec(struct rte_swx_pipeline *p)
3784 {
3785         struct thread *t = &p->threads[p->thread_id];
3786         struct instruction *ip = t->ip;
3787
3788         TRACE("[Thread %2u] mov\n",
3789               p->thread_id);
3790
3791         MOV(t, ip);
3792
3793         /* Thread. */
3794         thread_ip_inc(p);
3795 }
3796
3797 static inline void
3798 instr_mov_mh_exec(struct rte_swx_pipeline *p)
3799 {
3800         struct thread *t = &p->threads[p->thread_id];
3801         struct instruction *ip = t->ip;
3802
3803         TRACE("[Thread %2u] mov (mh)\n",
3804               p->thread_id);
3805
3806         MOV_MH(t, ip);
3807
3808         /* Thread. */
3809         thread_ip_inc(p);
3810 }
3811
3812 static inline void
3813 instr_mov_hm_exec(struct rte_swx_pipeline *p)
3814 {
3815         struct thread *t = &p->threads[p->thread_id];
3816         struct instruction *ip = t->ip;
3817
3818         TRACE("[Thread %2u] mov (hm)\n",
3819               p->thread_id);
3820
3821         MOV_HM(t, ip);
3822
3823         /* Thread. */
3824         thread_ip_inc(p);
3825 }
3826
3827 static inline void
3828 instr_mov_hh_exec(struct rte_swx_pipeline *p)
3829 {
3830         struct thread *t = &p->threads[p->thread_id];
3831         struct instruction *ip = t->ip;
3832
3833         TRACE("[Thread %2u] mov (hh)\n",
3834               p->thread_id);
3835
3836         MOV_HH(t, ip);
3837
3838         /* Thread. */
3839         thread_ip_inc(p);
3840 }
3841
3842 static inline void
3843 instr_mov_i_exec(struct rte_swx_pipeline *p)
3844 {
3845         struct thread *t = &p->threads[p->thread_id];
3846         struct instruction *ip = t->ip;
3847
3848         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
3849               p->thread_id,
3850               ip->mov.src_val);
3851
3852         MOV_I(t, ip);
3853
3854         /* Thread. */
3855         thread_ip_inc(p);
3856 }
3857
3858 /*
3859  * dma.
3860  */
3861 static inline void
3862 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3863
3864 static inline void
3865 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3866 {
3867         struct thread *t = &p->threads[p->thread_id];
3868         struct instruction *ip = t->ip;
3869         uint8_t *action_data = t->structs[0];
3870         uint64_t valid_headers = t->valid_headers;
3871         uint32_t i;
3872
3873         for (i = 0; i < n_dma; i++) {
3874                 uint32_t header_id = ip->dma.dst.header_id[i];
3875                 uint32_t struct_id = ip->dma.dst.struct_id[i];
3876                 uint32_t offset = ip->dma.src.offset[i];
3877                 uint32_t n_bytes = ip->dma.n_bytes[i];
3878
3879                 struct header_runtime *h = &t->headers[header_id];
3880                 uint8_t *h_ptr0 = h->ptr0;
3881                 uint8_t *h_ptr = t->structs[struct_id];
3882
3883                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3884                         h_ptr : h_ptr0;
3885                 void *src = &action_data[offset];
3886
3887                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3888
3889                 /* Headers. */
3890                 memcpy(dst, src, n_bytes);
3891                 t->structs[struct_id] = dst;
3892                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3893         }
3894
3895         t->valid_headers = valid_headers;
3896 }
3897
3898 static inline void
3899 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3900 {
3901         __instr_dma_ht_exec(p, 1);
3902
3903         /* Thread. */
3904         thread_ip_inc(p);
3905 }
3906
3907 static inline void
3908 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3909 {
3910         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3911               p->thread_id);
3912
3913         __instr_dma_ht_exec(p, 2);
3914
3915         /* Thread. */
3916         thread_ip_inc(p);
3917 }
3918
3919 static inline void
3920 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3921 {
3922         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3923               p->thread_id);
3924
3925         __instr_dma_ht_exec(p, 3);
3926
3927         /* Thread. */
3928         thread_ip_inc(p);
3929 }
3930
3931 static inline void
3932 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3933 {
3934         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3935               p->thread_id);
3936
3937         __instr_dma_ht_exec(p, 4);
3938
3939         /* Thread. */
3940         thread_ip_inc(p);
3941 }
3942
3943 static inline void
3944 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3945 {
3946         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3947               p->thread_id);
3948
3949         __instr_dma_ht_exec(p, 5);
3950
3951         /* Thread. */
3952         thread_ip_inc(p);
3953 }
3954
3955 static inline void
3956 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3957 {
3958         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3959               p->thread_id);
3960
3961         __instr_dma_ht_exec(p, 6);
3962
3963         /* Thread. */
3964         thread_ip_inc(p);
3965 }
3966
3967 static inline void
3968 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3969 {
3970         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3971               p->thread_id);
3972
3973         __instr_dma_ht_exec(p, 7);
3974
3975         /* Thread. */
3976         thread_ip_inc(p);
3977 }
3978
3979 static inline void
3980 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3981 {
3982         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3983               p->thread_id);
3984
3985         __instr_dma_ht_exec(p, 8);
3986
3987         /* Thread. */
3988         thread_ip_inc(p);
3989 }
3990
3991 /*
3992  * alu.
3993  */
3994 static int
3995 instr_alu_add_translate(struct rte_swx_pipeline *p,
3996                         struct action *action,
3997                         char **tokens,
3998                         int n_tokens,
3999                         struct instruction *instr,
4000                         struct instruction_data *data __rte_unused)
4001 {
4002         char *dst = tokens[1], *src = tokens[2];
4003         struct field *fdst, *fsrc;
4004         uint64_t src_val;
4005         uint32_t dst_struct_id = 0, src_struct_id = 0;
4006
4007         CHECK(n_tokens == 3, EINVAL);
4008
4009         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4010         CHECK(fdst, EINVAL);
4011         CHECK(!fdst->var_size, EINVAL);
4012
4013         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
4014         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4015         if (fsrc) {
4016                 CHECK(!fsrc->var_size, EINVAL);
4017
4018                 instr->type = INSTR_ALU_ADD;
4019                 if (dst[0] == 'h' && src[0] != 'h')
4020                         instr->type = INSTR_ALU_ADD_HM;
4021                 if (dst[0] != 'h' && src[0] == 'h')
4022                         instr->type = INSTR_ALU_ADD_MH;
4023                 if (dst[0] == 'h' && src[0] == 'h')
4024                         instr->type = INSTR_ALU_ADD_HH;
4025
4026                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4027                 instr->alu.dst.n_bits = fdst->n_bits;
4028                 instr->alu.dst.offset = fdst->offset / 8;
4029                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4030                 instr->alu.src.n_bits = fsrc->n_bits;
4031                 instr->alu.src.offset = fsrc->offset / 8;
4032                 return 0;
4033         }
4034
4035         /* ADD_MI, ADD_HI. */
4036         src_val = strtoull(src, &src, 0);
4037         CHECK(!src[0], EINVAL);
4038
4039         instr->type = INSTR_ALU_ADD_MI;
4040         if (dst[0] == 'h')
4041                 instr->type = INSTR_ALU_ADD_HI;
4042
4043         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4044         instr->alu.dst.n_bits = fdst->n_bits;
4045         instr->alu.dst.offset = fdst->offset / 8;
4046         instr->alu.src_val = src_val;
4047         return 0;
4048 }
4049
4050 static int
4051 instr_alu_sub_translate(struct rte_swx_pipeline *p,
4052                         struct action *action,
4053                         char **tokens,
4054                         int n_tokens,
4055                         struct instruction *instr,
4056                         struct instruction_data *data __rte_unused)
4057 {
4058         char *dst = tokens[1], *src = tokens[2];
4059         struct field *fdst, *fsrc;
4060         uint64_t src_val;
4061         uint32_t dst_struct_id = 0, src_struct_id = 0;
4062
4063         CHECK(n_tokens == 3, EINVAL);
4064
4065         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4066         CHECK(fdst, EINVAL);
4067         CHECK(!fdst->var_size, EINVAL);
4068
4069         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
4070         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4071         if (fsrc) {
4072                 CHECK(!fsrc->var_size, EINVAL);
4073
4074                 instr->type = INSTR_ALU_SUB;
4075                 if (dst[0] == 'h' && src[0] != 'h')
4076                         instr->type = INSTR_ALU_SUB_HM;
4077                 if (dst[0] != 'h' && src[0] == 'h')
4078                         instr->type = INSTR_ALU_SUB_MH;
4079                 if (dst[0] == 'h' && src[0] == 'h')
4080                         instr->type = INSTR_ALU_SUB_HH;
4081
4082                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4083                 instr->alu.dst.n_bits = fdst->n_bits;
4084                 instr->alu.dst.offset = fdst->offset / 8;
4085                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4086                 instr->alu.src.n_bits = fsrc->n_bits;
4087                 instr->alu.src.offset = fsrc->offset / 8;
4088                 return 0;
4089         }
4090
4091         /* SUB_MI, SUB_HI. */
4092         src_val = strtoull(src, &src, 0);
4093         CHECK(!src[0], EINVAL);
4094
4095         instr->type = INSTR_ALU_SUB_MI;
4096         if (dst[0] == 'h')
4097                 instr->type = INSTR_ALU_SUB_HI;
4098
4099         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4100         instr->alu.dst.n_bits = fdst->n_bits;
4101         instr->alu.dst.offset = fdst->offset / 8;
4102         instr->alu.src_val = src_val;
4103         return 0;
4104 }
4105
4106 static int
4107 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
4108                           struct action *action __rte_unused,
4109                           char **tokens,
4110                           int n_tokens,
4111                           struct instruction *instr,
4112                           struct instruction_data *data __rte_unused)
4113 {
4114         char *dst = tokens[1], *src = tokens[2];
4115         struct header *hdst, *hsrc;
4116         struct field *fdst, *fsrc;
4117
4118         CHECK(n_tokens == 3, EINVAL);
4119
4120         fdst = header_field_parse(p, dst, &hdst);
4121         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4122         CHECK(!fdst->var_size, EINVAL);
4123
4124         /* CKADD_FIELD. */
4125         fsrc = header_field_parse(p, src, &hsrc);
4126         if (fsrc) {
4127                 CHECK(!fsrc->var_size, EINVAL);
4128
4129                 instr->type = INSTR_ALU_CKADD_FIELD;
4130                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4131                 instr->alu.dst.n_bits = fdst->n_bits;
4132                 instr->alu.dst.offset = fdst->offset / 8;
4133                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4134                 instr->alu.src.n_bits = fsrc->n_bits;
4135                 instr->alu.src.offset = fsrc->offset / 8;
4136                 return 0;
4137         }
4138
4139         /* CKADD_STRUCT, CKADD_STRUCT20. */
4140         hsrc = header_parse(p, src);
4141         CHECK(hsrc, EINVAL);
4142         CHECK(!hsrc->st->var_size, EINVAL);
4143
4144         instr->type = INSTR_ALU_CKADD_STRUCT;
4145         if ((hsrc->st->n_bits / 8) == 20)
4146                 instr->type = INSTR_ALU_CKADD_STRUCT20;
4147
4148         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4149         instr->alu.dst.n_bits = fdst->n_bits;
4150         instr->alu.dst.offset = fdst->offset / 8;
4151         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4152         instr->alu.src.n_bits = hsrc->st->n_bits;
4153         instr->alu.src.offset = 0; /* Unused. */
4154         return 0;
4155 }
4156
4157 static int
4158 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
4159                           struct action *action __rte_unused,
4160                           char **tokens,
4161                           int n_tokens,
4162                           struct instruction *instr,
4163                           struct instruction_data *data __rte_unused)
4164 {
4165         char *dst = tokens[1], *src = tokens[2];
4166         struct header *hdst, *hsrc;
4167         struct field *fdst, *fsrc;
4168
4169         CHECK(n_tokens == 3, EINVAL);
4170
4171         fdst = header_field_parse(p, dst, &hdst);
4172         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4173         CHECK(!fdst->var_size, EINVAL);
4174
4175         fsrc = header_field_parse(p, src, &hsrc);
4176         CHECK(fsrc, EINVAL);
4177         CHECK(!fsrc->var_size, EINVAL);
4178
4179         instr->type = INSTR_ALU_CKSUB_FIELD;
4180         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4181         instr->alu.dst.n_bits = fdst->n_bits;
4182         instr->alu.dst.offset = fdst->offset / 8;
4183         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4184         instr->alu.src.n_bits = fsrc->n_bits;
4185         instr->alu.src.offset = fsrc->offset / 8;
4186         return 0;
4187 }
4188
4189 static int
4190 instr_alu_shl_translate(struct rte_swx_pipeline *p,
4191                         struct action *action,
4192                         char **tokens,
4193                         int n_tokens,
4194                         struct instruction *instr,
4195                         struct instruction_data *data __rte_unused)
4196 {
4197         char *dst = tokens[1], *src = tokens[2];
4198         struct field *fdst, *fsrc;
4199         uint64_t src_val;
4200         uint32_t dst_struct_id = 0, src_struct_id = 0;
4201
4202         CHECK(n_tokens == 3, EINVAL);
4203
4204         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4205         CHECK(fdst, EINVAL);
4206         CHECK(!fdst->var_size, EINVAL);
4207
4208         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
4209         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4210         if (fsrc) {
4211                 CHECK(!fsrc->var_size, EINVAL);
4212
4213                 instr->type = INSTR_ALU_SHL;
4214                 if (dst[0] == 'h' && src[0] != 'h')
4215                         instr->type = INSTR_ALU_SHL_HM;
4216                 if (dst[0] != 'h' && src[0] == 'h')
4217                         instr->type = INSTR_ALU_SHL_MH;
4218                 if (dst[0] == 'h' && src[0] == 'h')
4219                         instr->type = INSTR_ALU_SHL_HH;
4220
4221                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4222                 instr->alu.dst.n_bits = fdst->n_bits;
4223                 instr->alu.dst.offset = fdst->offset / 8;
4224                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4225                 instr->alu.src.n_bits = fsrc->n_bits;
4226                 instr->alu.src.offset = fsrc->offset / 8;
4227                 return 0;
4228         }
4229
4230         /* SHL_MI, SHL_HI. */
4231         src_val = strtoull(src, &src, 0);
4232         CHECK(!src[0], EINVAL);
4233
4234         instr->type = INSTR_ALU_SHL_MI;
4235         if (dst[0] == 'h')
4236                 instr->type = INSTR_ALU_SHL_HI;
4237
4238         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4239         instr->alu.dst.n_bits = fdst->n_bits;
4240         instr->alu.dst.offset = fdst->offset / 8;
4241         instr->alu.src_val = src_val;
4242         return 0;
4243 }
4244
4245 static int
4246 instr_alu_shr_translate(struct rte_swx_pipeline *p,
4247                         struct action *action,
4248                         char **tokens,
4249                         int n_tokens,
4250                         struct instruction *instr,
4251                         struct instruction_data *data __rte_unused)
4252 {
4253         char *dst = tokens[1], *src = tokens[2];
4254         struct field *fdst, *fsrc;
4255         uint64_t src_val;
4256         uint32_t dst_struct_id = 0, src_struct_id = 0;
4257
4258         CHECK(n_tokens == 3, EINVAL);
4259
4260         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4261         CHECK(fdst, EINVAL);
4262         CHECK(!fdst->var_size, EINVAL);
4263
4264         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
4265         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4266         if (fsrc) {
4267                 CHECK(!fsrc->var_size, EINVAL);
4268
4269                 instr->type = INSTR_ALU_SHR;
4270                 if (dst[0] == 'h' && src[0] != 'h')
4271                         instr->type = INSTR_ALU_SHR_HM;
4272                 if (dst[0] != 'h' && src[0] == 'h')
4273                         instr->type = INSTR_ALU_SHR_MH;
4274                 if (dst[0] == 'h' && src[0] == 'h')
4275                         instr->type = INSTR_ALU_SHR_HH;
4276
4277                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4278                 instr->alu.dst.n_bits = fdst->n_bits;
4279                 instr->alu.dst.offset = fdst->offset / 8;
4280                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4281                 instr->alu.src.n_bits = fsrc->n_bits;
4282                 instr->alu.src.offset = fsrc->offset / 8;
4283                 return 0;
4284         }
4285
4286         /* SHR_MI, SHR_HI. */
4287         src_val = strtoull(src, &src, 0);
4288         CHECK(!src[0], EINVAL);
4289
4290         instr->type = INSTR_ALU_SHR_MI;
4291         if (dst[0] == 'h')
4292                 instr->type = INSTR_ALU_SHR_HI;
4293
4294         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4295         instr->alu.dst.n_bits = fdst->n_bits;
4296         instr->alu.dst.offset = fdst->offset / 8;
4297         instr->alu.src_val = src_val;
4298         return 0;
4299 }
4300
4301 static int
4302 instr_alu_and_translate(struct rte_swx_pipeline *p,
4303                         struct action *action,
4304                         char **tokens,
4305                         int n_tokens,
4306                         struct instruction *instr,
4307                         struct instruction_data *data __rte_unused)
4308 {
4309         char *dst = tokens[1], *src = tokens[2];
4310         struct field *fdst, *fsrc;
4311         uint64_t src_val;
4312         uint32_t dst_struct_id = 0, src_struct_id = 0;
4313
4314         CHECK(n_tokens == 3, EINVAL);
4315
4316         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4317         CHECK(fdst, EINVAL);
4318         CHECK(!fdst->var_size, EINVAL);
4319
4320         /* AND, AND_MH, AND_HM, AND_HH. */
4321         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4322         if (fsrc) {
4323                 CHECK(!fsrc->var_size, EINVAL);
4324
4325                 instr->type = INSTR_ALU_AND;
4326                 if (dst[0] != 'h' && src[0] == 'h')
4327                         instr->type = INSTR_ALU_AND_MH;
4328                 if (dst[0] == 'h' && src[0] != 'h')
4329                         instr->type = INSTR_ALU_AND_HM;
4330                 if (dst[0] == 'h' && src[0] == 'h')
4331                         instr->type = INSTR_ALU_AND_HH;
4332
4333                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4334                 instr->alu.dst.n_bits = fdst->n_bits;
4335                 instr->alu.dst.offset = fdst->offset / 8;
4336                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4337                 instr->alu.src.n_bits = fsrc->n_bits;
4338                 instr->alu.src.offset = fsrc->offset / 8;
4339                 return 0;
4340         }
4341
4342         /* AND_I. */
4343         src_val = strtoull(src, &src, 0);
4344         CHECK(!src[0], EINVAL);
4345
4346         if (dst[0] == 'h')
4347                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4348
4349         instr->type = INSTR_ALU_AND_I;
4350         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4351         instr->alu.dst.n_bits = fdst->n_bits;
4352         instr->alu.dst.offset = fdst->offset / 8;
4353         instr->alu.src_val = src_val;
4354         return 0;
4355 }
4356
4357 static int
4358 instr_alu_or_translate(struct rte_swx_pipeline *p,
4359                        struct action *action,
4360                        char **tokens,
4361                        int n_tokens,
4362                        struct instruction *instr,
4363                        struct instruction_data *data __rte_unused)
4364 {
4365         char *dst = tokens[1], *src = tokens[2];
4366         struct field *fdst, *fsrc;
4367         uint64_t src_val;
4368         uint32_t dst_struct_id = 0, src_struct_id = 0;
4369
4370         CHECK(n_tokens == 3, EINVAL);
4371
4372         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4373         CHECK(fdst, EINVAL);
4374         CHECK(!fdst->var_size, EINVAL);
4375
4376         /* OR, OR_MH, OR_HM, OR_HH. */
4377         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4378         if (fsrc) {
4379                 CHECK(!fsrc->var_size, EINVAL);
4380
4381                 instr->type = INSTR_ALU_OR;
4382                 if (dst[0] != 'h' && src[0] == 'h')
4383                         instr->type = INSTR_ALU_OR_MH;
4384                 if (dst[0] == 'h' && src[0] != 'h')
4385                         instr->type = INSTR_ALU_OR_HM;
4386                 if (dst[0] == 'h' && src[0] == 'h')
4387                         instr->type = INSTR_ALU_OR_HH;
4388
4389                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4390                 instr->alu.dst.n_bits = fdst->n_bits;
4391                 instr->alu.dst.offset = fdst->offset / 8;
4392                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4393                 instr->alu.src.n_bits = fsrc->n_bits;
4394                 instr->alu.src.offset = fsrc->offset / 8;
4395                 return 0;
4396         }
4397
4398         /* OR_I. */
4399         src_val = strtoull(src, &src, 0);
4400         CHECK(!src[0], EINVAL);
4401
4402         if (dst[0] == 'h')
4403                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4404
4405         instr->type = INSTR_ALU_OR_I;
4406         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4407         instr->alu.dst.n_bits = fdst->n_bits;
4408         instr->alu.dst.offset = fdst->offset / 8;
4409         instr->alu.src_val = src_val;
4410         return 0;
4411 }
4412
4413 static int
4414 instr_alu_xor_translate(struct rte_swx_pipeline *p,
4415                         struct action *action,
4416                         char **tokens,
4417                         int n_tokens,
4418                         struct instruction *instr,
4419                         struct instruction_data *data __rte_unused)
4420 {
4421         char *dst = tokens[1], *src = tokens[2];
4422         struct field *fdst, *fsrc;
4423         uint64_t src_val;
4424         uint32_t dst_struct_id = 0, src_struct_id = 0;
4425
4426         CHECK(n_tokens == 3, EINVAL);
4427
4428         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4429         CHECK(fdst, EINVAL);
4430         CHECK(!fdst->var_size, EINVAL);
4431
4432         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
4433         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4434         if (fsrc) {
4435                 CHECK(!fsrc->var_size, EINVAL);
4436
4437                 instr->type = INSTR_ALU_XOR;
4438                 if (dst[0] != 'h' && src[0] == 'h')
4439                         instr->type = INSTR_ALU_XOR_MH;
4440                 if (dst[0] == 'h' && src[0] != 'h')
4441                         instr->type = INSTR_ALU_XOR_HM;
4442                 if (dst[0] == 'h' && src[0] == 'h')
4443                         instr->type = INSTR_ALU_XOR_HH;
4444
4445                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4446                 instr->alu.dst.n_bits = fdst->n_bits;
4447                 instr->alu.dst.offset = fdst->offset / 8;
4448                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4449                 instr->alu.src.n_bits = fsrc->n_bits;
4450                 instr->alu.src.offset = fsrc->offset / 8;
4451                 return 0;
4452         }
4453
4454         /* XOR_I. */
4455         src_val = strtoull(src, &src, 0);
4456         CHECK(!src[0], EINVAL);
4457
4458         if (dst[0] == 'h')
4459                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4460
4461         instr->type = INSTR_ALU_XOR_I;
4462         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4463         instr->alu.dst.n_bits = fdst->n_bits;
4464         instr->alu.dst.offset = fdst->offset / 8;
4465         instr->alu.src_val = src_val;
4466         return 0;
4467 }
4468
4469 static inline void
4470 instr_alu_add_exec(struct rte_swx_pipeline *p)
4471 {
4472         struct thread *t = &p->threads[p->thread_id];
4473         struct instruction *ip = t->ip;
4474
4475         TRACE("[Thread %2u] add\n", p->thread_id);
4476
4477         /* Structs. */
4478         ALU(t, ip, +);
4479
4480         /* Thread. */
4481         thread_ip_inc(p);
4482 }
4483
4484 static inline void
4485 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
4486 {
4487         struct thread *t = &p->threads[p->thread_id];
4488         struct instruction *ip = t->ip;
4489
4490         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
4491
4492         /* Structs. */
4493         ALU_MH(t, ip, +);
4494
4495         /* Thread. */
4496         thread_ip_inc(p);
4497 }
4498
4499 static inline void
4500 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
4501 {
4502         struct thread *t = &p->threads[p->thread_id];
4503         struct instruction *ip = t->ip;
4504
4505         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
4506
4507         /* Structs. */
4508         ALU_HM(t, ip, +);
4509
4510         /* Thread. */
4511         thread_ip_inc(p);
4512 }
4513
4514 static inline void
4515 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
4516 {
4517         struct thread *t = &p->threads[p->thread_id];
4518         struct instruction *ip = t->ip;
4519
4520         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
4521
4522         /* Structs. */
4523         ALU_HH(t, ip, +);
4524
4525         /* Thread. */
4526         thread_ip_inc(p);
4527 }
4528
4529 static inline void
4530 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
4531 {
4532         struct thread *t = &p->threads[p->thread_id];
4533         struct instruction *ip = t->ip;
4534
4535         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
4536
4537         /* Structs. */
4538         ALU_MI(t, ip, +);
4539
4540         /* Thread. */
4541         thread_ip_inc(p);
4542 }
4543
4544 static inline void
4545 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
4546 {
4547         struct thread *t = &p->threads[p->thread_id];
4548         struct instruction *ip = t->ip;
4549
4550         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
4551
4552         /* Structs. */
4553         ALU_HI(t, ip, +);
4554
4555         /* Thread. */
4556         thread_ip_inc(p);
4557 }
4558
4559 static inline void
4560 instr_alu_sub_exec(struct rte_swx_pipeline *p)
4561 {
4562         struct thread *t = &p->threads[p->thread_id];
4563         struct instruction *ip = t->ip;
4564
4565         TRACE("[Thread %2u] sub\n", p->thread_id);
4566
4567         /* Structs. */
4568         ALU(t, ip, -);
4569
4570         /* Thread. */
4571         thread_ip_inc(p);
4572 }
4573
4574 static inline void
4575 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
4576 {
4577         struct thread *t = &p->threads[p->thread_id];
4578         struct instruction *ip = t->ip;
4579
4580         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
4581
4582         /* Structs. */
4583         ALU_MH(t, ip, -);
4584
4585         /* Thread. */
4586         thread_ip_inc(p);
4587 }
4588
4589 static inline void
4590 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4591 {
4592         struct thread *t = &p->threads[p->thread_id];
4593         struct instruction *ip = t->ip;
4594
4595         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4596
4597         /* Structs. */
4598         ALU_HM(t, ip, -);
4599
4600         /* Thread. */
4601         thread_ip_inc(p);
4602 }
4603
4604 static inline void
4605 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4606 {
4607         struct thread *t = &p->threads[p->thread_id];
4608         struct instruction *ip = t->ip;
4609
4610         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4611
4612         /* Structs. */
4613         ALU_HH(t, ip, -);
4614
4615         /* Thread. */
4616         thread_ip_inc(p);
4617 }
4618
4619 static inline void
4620 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4621 {
4622         struct thread *t = &p->threads[p->thread_id];
4623         struct instruction *ip = t->ip;
4624
4625         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4626
4627         /* Structs. */
4628         ALU_MI(t, ip, -);
4629
4630         /* Thread. */
4631         thread_ip_inc(p);
4632 }
4633
4634 static inline void
4635 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4636 {
4637         struct thread *t = &p->threads[p->thread_id];
4638         struct instruction *ip = t->ip;
4639
4640         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4641
4642         /* Structs. */
4643         ALU_HI(t, ip, -);
4644
4645         /* Thread. */
4646         thread_ip_inc(p);
4647 }
4648
4649 static inline void
4650 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4651 {
4652         struct thread *t = &p->threads[p->thread_id];
4653         struct instruction *ip = t->ip;
4654
4655         TRACE("[Thread %2u] shl\n", p->thread_id);
4656
4657         /* Structs. */
4658         ALU(t, ip, <<);
4659
4660         /* Thread. */
4661         thread_ip_inc(p);
4662 }
4663
4664 static inline void
4665 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4666 {
4667         struct thread *t = &p->threads[p->thread_id];
4668         struct instruction *ip = t->ip;
4669
4670         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4671
4672         /* Structs. */
4673         ALU_MH(t, ip, <<);
4674
4675         /* Thread. */
4676         thread_ip_inc(p);
4677 }
4678
4679 static inline void
4680 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4681 {
4682         struct thread *t = &p->threads[p->thread_id];
4683         struct instruction *ip = t->ip;
4684
4685         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4686
4687         /* Structs. */
4688         ALU_HM(t, ip, <<);
4689
4690         /* Thread. */
4691         thread_ip_inc(p);
4692 }
4693
4694 static inline void
4695 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4696 {
4697         struct thread *t = &p->threads[p->thread_id];
4698         struct instruction *ip = t->ip;
4699
4700         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4701
4702         /* Structs. */
4703         ALU_HH(t, ip, <<);
4704
4705         /* Thread. */
4706         thread_ip_inc(p);
4707 }
4708
4709 static inline void
4710 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4711 {
4712         struct thread *t = &p->threads[p->thread_id];
4713         struct instruction *ip = t->ip;
4714
4715         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4716
4717         /* Structs. */
4718         ALU_MI(t, ip, <<);
4719
4720         /* Thread. */
4721         thread_ip_inc(p);
4722 }
4723
4724 static inline void
4725 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4726 {
4727         struct thread *t = &p->threads[p->thread_id];
4728         struct instruction *ip = t->ip;
4729
4730         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4731
4732         /* Structs. */
4733         ALU_HI(t, ip, <<);
4734
4735         /* Thread. */
4736         thread_ip_inc(p);
4737 }
4738
4739 static inline void
4740 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4741 {
4742         struct thread *t = &p->threads[p->thread_id];
4743         struct instruction *ip = t->ip;
4744
4745         TRACE("[Thread %2u] shr\n", p->thread_id);
4746
4747         /* Structs. */
4748         ALU(t, ip, >>);
4749
4750         /* Thread. */
4751         thread_ip_inc(p);
4752 }
4753
4754 static inline void
4755 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4756 {
4757         struct thread *t = &p->threads[p->thread_id];
4758         struct instruction *ip = t->ip;
4759
4760         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4761
4762         /* Structs. */
4763         ALU_MH(t, ip, >>);
4764
4765         /* Thread. */
4766         thread_ip_inc(p);
4767 }
4768
4769 static inline void
4770 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4771 {
4772         struct thread *t = &p->threads[p->thread_id];
4773         struct instruction *ip = t->ip;
4774
4775         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4776
4777         /* Structs. */
4778         ALU_HM(t, ip, >>);
4779
4780         /* Thread. */
4781         thread_ip_inc(p);
4782 }
4783
4784 static inline void
4785 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4786 {
4787         struct thread *t = &p->threads[p->thread_id];
4788         struct instruction *ip = t->ip;
4789
4790         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4791
4792         /* Structs. */
4793         ALU_HH(t, ip, >>);
4794
4795         /* Thread. */
4796         thread_ip_inc(p);
4797 }
4798
4799 static inline void
4800 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4801 {
4802         struct thread *t = &p->threads[p->thread_id];
4803         struct instruction *ip = t->ip;
4804
4805         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4806
4807         /* Structs. */
4808         ALU_MI(t, ip, >>);
4809
4810         /* Thread. */
4811         thread_ip_inc(p);
4812 }
4813
4814 static inline void
4815 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4816 {
4817         struct thread *t = &p->threads[p->thread_id];
4818         struct instruction *ip = t->ip;
4819
4820         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4821
4822         /* Structs. */
4823         ALU_HI(t, ip, >>);
4824
4825         /* Thread. */
4826         thread_ip_inc(p);
4827 }
4828
4829 static inline void
4830 instr_alu_and_exec(struct rte_swx_pipeline *p)
4831 {
4832         struct thread *t = &p->threads[p->thread_id];
4833         struct instruction *ip = t->ip;
4834
4835         TRACE("[Thread %2u] and\n", p->thread_id);
4836
4837         /* Structs. */
4838         ALU(t, ip, &);
4839
4840         /* Thread. */
4841         thread_ip_inc(p);
4842 }
4843
4844 static inline void
4845 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
4846 {
4847         struct thread *t = &p->threads[p->thread_id];
4848         struct instruction *ip = t->ip;
4849
4850         TRACE("[Thread %2u] and (mh)\n", p->thread_id);
4851
4852         /* Structs. */
4853         ALU_MH(t, ip, &);
4854
4855         /* Thread. */
4856         thread_ip_inc(p);
4857 }
4858
4859 static inline void
4860 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
4861 {
4862         struct thread *t = &p->threads[p->thread_id];
4863         struct instruction *ip = t->ip;
4864
4865         TRACE("[Thread %2u] and (hm)\n", p->thread_id);
4866
4867         /* Structs. */
4868         ALU_HM_FAST(t, ip, &);
4869
4870         /* Thread. */
4871         thread_ip_inc(p);
4872 }
4873
4874 static inline void
4875 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
4876 {
4877         struct thread *t = &p->threads[p->thread_id];
4878         struct instruction *ip = t->ip;
4879
4880         TRACE("[Thread %2u] and (hh)\n", p->thread_id);
4881
4882         /* Structs. */
4883         ALU_HH_FAST(t, ip, &);
4884
4885         /* Thread. */
4886         thread_ip_inc(p);
4887 }
4888
4889 static inline void
4890 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4891 {
4892         struct thread *t = &p->threads[p->thread_id];
4893         struct instruction *ip = t->ip;
4894
4895         TRACE("[Thread %2u] and (i)\n", p->thread_id);
4896
4897         /* Structs. */
4898         ALU_I(t, ip, &);
4899
4900         /* Thread. */
4901         thread_ip_inc(p);
4902 }
4903
4904 static inline void
4905 instr_alu_or_exec(struct rte_swx_pipeline *p)
4906 {
4907         struct thread *t = &p->threads[p->thread_id];
4908         struct instruction *ip = t->ip;
4909
4910         TRACE("[Thread %2u] or\n", p->thread_id);
4911
4912         /* Structs. */
4913         ALU(t, ip, |);
4914
4915         /* Thread. */
4916         thread_ip_inc(p);
4917 }
4918
4919 static inline void
4920 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
4921 {
4922         struct thread *t = &p->threads[p->thread_id];
4923         struct instruction *ip = t->ip;
4924
4925         TRACE("[Thread %2u] or (mh)\n", p->thread_id);
4926
4927         /* Structs. */
4928         ALU_MH(t, ip, |);
4929
4930         /* Thread. */
4931         thread_ip_inc(p);
4932 }
4933
4934 static inline void
4935 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
4936 {
4937         struct thread *t = &p->threads[p->thread_id];
4938         struct instruction *ip = t->ip;
4939
4940         TRACE("[Thread %2u] or (hm)\n", p->thread_id);
4941
4942         /* Structs. */
4943         ALU_HM_FAST(t, ip, |);
4944
4945         /* Thread. */
4946         thread_ip_inc(p);
4947 }
4948
4949 static inline void
4950 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
4951 {
4952         struct thread *t = &p->threads[p->thread_id];
4953         struct instruction *ip = t->ip;
4954
4955         TRACE("[Thread %2u] or (hh)\n", p->thread_id);
4956
4957         /* Structs. */
4958         ALU_HH_FAST(t, ip, |);
4959
4960         /* Thread. */
4961         thread_ip_inc(p);
4962 }
4963
4964 static inline void
4965 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4966 {
4967         struct thread *t = &p->threads[p->thread_id];
4968         struct instruction *ip = t->ip;
4969
4970         TRACE("[Thread %2u] or (i)\n", p->thread_id);
4971
4972         /* Structs. */
4973         ALU_I(t, ip, |);
4974
4975         /* Thread. */
4976         thread_ip_inc(p);
4977 }
4978
4979 static inline void
4980 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4981 {
4982         struct thread *t = &p->threads[p->thread_id];
4983         struct instruction *ip = t->ip;
4984
4985         TRACE("[Thread %2u] xor\n", p->thread_id);
4986
4987         /* Structs. */
4988         ALU(t, ip, ^);
4989
4990         /* Thread. */
4991         thread_ip_inc(p);
4992 }
4993
4994 static inline void
4995 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
4996 {
4997         struct thread *t = &p->threads[p->thread_id];
4998         struct instruction *ip = t->ip;
4999
5000         TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
5001
5002         /* Structs. */
5003         ALU_MH(t, ip, ^);
5004
5005         /* Thread. */
5006         thread_ip_inc(p);
5007 }
5008
5009 static inline void
5010 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
5011 {
5012         struct thread *t = &p->threads[p->thread_id];
5013         struct instruction *ip = t->ip;
5014
5015         TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
5016
5017         /* Structs. */
5018         ALU_HM_FAST(t, ip, ^);
5019
5020         /* Thread. */
5021         thread_ip_inc(p);
5022 }
5023
5024 static inline void
5025 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
5026 {
5027         struct thread *t = &p->threads[p->thread_id];
5028         struct instruction *ip = t->ip;
5029
5030         TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
5031
5032         /* Structs. */
5033         ALU_HH_FAST(t, ip, ^);
5034
5035         /* Thread. */
5036         thread_ip_inc(p);
5037 }
5038
5039 static inline void
5040 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
5041 {
5042         struct thread *t = &p->threads[p->thread_id];
5043         struct instruction *ip = t->ip;
5044
5045         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
5046
5047         /* Structs. */
5048         ALU_I(t, ip, ^);
5049
5050         /* Thread. */
5051         thread_ip_inc(p);
5052 }
5053
5054 static inline void
5055 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
5056 {
5057         struct thread *t = &p->threads[p->thread_id];
5058         struct instruction *ip = t->ip;
5059         uint8_t *dst_struct, *src_struct;
5060         uint16_t *dst16_ptr, dst;
5061         uint64_t *src64_ptr, src64, src64_mask, src;
5062         uint64_t r;
5063
5064         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
5065
5066         /* Structs. */
5067         dst_struct = t->structs[ip->alu.dst.struct_id];
5068         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5069         dst = *dst16_ptr;
5070
5071         src_struct = t->structs[ip->alu.src.struct_id];
5072         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
5073         src64 = *src64_ptr;
5074         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
5075         src = src64 & src64_mask;
5076
5077         r = dst;
5078         r = ~r & 0xFFFF;
5079
5080         /* The first input (r) is a 16-bit number. The second and the third
5081          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
5082          * three numbers (output r) is a 34-bit number.
5083          */
5084         r += (src >> 32) + (src & 0xFFFFFFFF);
5085
5086         /* The first input is a 16-bit number. The second input is an 18-bit
5087          * number. In the worst case scenario, the sum of the two numbers is a
5088          * 19-bit number.
5089          */
5090         r = (r & 0xFFFF) + (r >> 16);
5091
5092         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5093          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
5094          */
5095         r = (r & 0xFFFF) + (r >> 16);
5096
5097         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5098          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5099          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
5100          * therefore the output r is always a 16-bit number.
5101          */
5102         r = (r & 0xFFFF) + (r >> 16);
5103
5104         r = ~r & 0xFFFF;
5105         r = r ? r : 0xFFFF;
5106
5107         *dst16_ptr = (uint16_t)r;
5108
5109         /* Thread. */
5110         thread_ip_inc(p);
5111 }
5112
5113 static inline void
5114 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
5115 {
5116         struct thread *t = &p->threads[p->thread_id];
5117         struct instruction *ip = t->ip;
5118         uint8_t *dst_struct, *src_struct;
5119         uint16_t *dst16_ptr, dst;
5120         uint64_t *src64_ptr, src64, src64_mask, src;
5121         uint64_t r;
5122
5123         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
5124
5125         /* Structs. */
5126         dst_struct = t->structs[ip->alu.dst.struct_id];
5127         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5128         dst = *dst16_ptr;
5129
5130         src_struct = t->structs[ip->alu.src.struct_id];
5131         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
5132         src64 = *src64_ptr;
5133         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
5134         src = src64 & src64_mask;
5135
5136         r = dst;
5137         r = ~r & 0xFFFF;
5138
5139         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
5140          * the following sequence of operations in 2's complement arithmetic:
5141          *    a '- b = (a - b) % 0xFFFF.
5142          *
5143          * In order to prevent an underflow for the below subtraction, in which
5144          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
5145          * minuend), we first add a multiple of the 0xFFFF modulus to the
5146          * minuend. The number we add to the minuend needs to be a 34-bit number
5147          * or higher, so for readability reasons we picked the 36-bit multiple.
5148          * We are effectively turning the 16-bit minuend into a 36-bit number:
5149          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
5150          */
5151         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
5152
5153         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
5154          * result (the output r) is a 36-bit number.
5155          */
5156         r -= (src >> 32) + (src & 0xFFFFFFFF);
5157
5158         /* The first input is a 16-bit number. The second input is a 20-bit
5159          * number. Their sum is a 21-bit number.
5160          */
5161         r = (r & 0xFFFF) + (r >> 16);
5162
5163         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5164          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
5165          */
5166         r = (r & 0xFFFF) + (r >> 16);
5167
5168         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5169          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5170          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5171          * generated, therefore the output r is always a 16-bit number.
5172          */
5173         r = (r & 0xFFFF) + (r >> 16);
5174
5175         r = ~r & 0xFFFF;
5176         r = r ? r : 0xFFFF;
5177
5178         *dst16_ptr = (uint16_t)r;
5179
5180         /* Thread. */
5181         thread_ip_inc(p);
5182 }
5183
5184 static inline void
5185 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
5186 {
5187         struct thread *t = &p->threads[p->thread_id];
5188         struct instruction *ip = t->ip;
5189         uint8_t *dst_struct, *src_struct;
5190         uint16_t *dst16_ptr;
5191         uint32_t *src32_ptr;
5192         uint64_t r0, r1;
5193
5194         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
5195
5196         /* Structs. */
5197         dst_struct = t->structs[ip->alu.dst.struct_id];
5198         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5199
5200         src_struct = t->structs[ip->alu.src.struct_id];
5201         src32_ptr = (uint32_t *)&src_struct[0];
5202
5203         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
5204         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
5205         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
5206         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
5207         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
5208
5209         /* The first input is a 16-bit number. The second input is a 19-bit
5210          * number. Their sum is a 20-bit number.
5211          */
5212         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5213
5214         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5215          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
5216          */
5217         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5218
5219         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5220          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5221          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
5222          * generated, therefore the output r is always a 16-bit number.
5223          */
5224         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5225
5226         r0 = ~r0 & 0xFFFF;
5227         r0 = r0 ? r0 : 0xFFFF;
5228
5229         *dst16_ptr = (uint16_t)r0;
5230
5231         /* Thread. */
5232         thread_ip_inc(p);
5233 }
5234
5235 static inline void
5236 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
5237 {
5238         struct thread *t = &p->threads[p->thread_id];
5239         struct instruction *ip = t->ip;
5240         uint8_t *dst_struct, *src_struct;
5241         uint16_t *dst16_ptr;
5242         uint32_t *src32_ptr;
5243         uint64_t r = 0;
5244         uint32_t i;
5245
5246         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
5247
5248         /* Structs. */
5249         dst_struct = t->structs[ip->alu.dst.struct_id];
5250         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5251
5252         src_struct = t->structs[ip->alu.src.struct_id];
5253         src32_ptr = (uint32_t *)&src_struct[0];
5254
5255         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
5256          * Therefore, in the worst case scenario, a 35-bit number is added to a
5257          * 16-bit number (the input r), so the output r is 36-bit number.
5258          */
5259         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
5260                 r += *src32_ptr;
5261
5262         /* The first input is a 16-bit number. The second input is a 20-bit
5263          * number. Their sum is a 21-bit number.
5264          */
5265         r = (r & 0xFFFF) + (r >> 16);
5266
5267         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5268          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
5269          */
5270         r = (r & 0xFFFF) + (r >> 16);
5271
5272         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5273          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5274          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5275          * generated, therefore the output r is always a 16-bit number.
5276          */
5277         r = (r & 0xFFFF) + (r >> 16);
5278
5279         r = ~r & 0xFFFF;
5280         r = r ? r : 0xFFFF;
5281
5282         *dst16_ptr = (uint16_t)r;
5283
5284         /* Thread. */
5285         thread_ip_inc(p);
5286 }
5287
5288 /*
5289  * Register array.
5290  */
5291 static struct regarray *
5292 regarray_find(struct rte_swx_pipeline *p, const char *name);
5293
5294 static int
5295 instr_regprefetch_translate(struct rte_swx_pipeline *p,
5296                       struct action *action,
5297                       char **tokens,
5298                       int n_tokens,
5299                       struct instruction *instr,
5300                       struct instruction_data *data __rte_unused)
5301 {
5302         char *regarray = tokens[1], *idx = tokens[2];
5303         struct regarray *r;
5304         struct field *fidx;
5305         uint32_t idx_struct_id, idx_val;
5306
5307         CHECK(n_tokens == 3, EINVAL);
5308
5309         r = regarray_find(p, regarray);
5310         CHECK(r, EINVAL);
5311
5312         /* REGPREFETCH_RH, REGPREFETCH_RM. */
5313         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5314         if (fidx) {
5315                 CHECK(!fidx->var_size, EINVAL);
5316
5317                 instr->type = INSTR_REGPREFETCH_RM;
5318                 if (idx[0] == 'h')
5319                         instr->type = INSTR_REGPREFETCH_RH;
5320
5321                 instr->regarray.regarray_id = r->id;
5322                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5323                 instr->regarray.idx.n_bits = fidx->n_bits;
5324                 instr->regarray.idx.offset = fidx->offset / 8;
5325                 instr->regarray.dstsrc_val = 0; /* Unused. */
5326                 return 0;
5327         }
5328
5329         /* REGPREFETCH_RI. */
5330         idx_val = strtoul(idx, &idx, 0);
5331         CHECK(!idx[0], EINVAL);
5332
5333         instr->type = INSTR_REGPREFETCH_RI;
5334         instr->regarray.regarray_id = r->id;
5335         instr->regarray.idx_val = idx_val;
5336         instr->regarray.dstsrc_val = 0; /* Unused. */
5337         return 0;
5338 }
5339
5340 static int
5341 instr_regrd_translate(struct rte_swx_pipeline *p,
5342                       struct action *action,
5343                       char **tokens,
5344                       int n_tokens,
5345                       struct instruction *instr,
5346                       struct instruction_data *data __rte_unused)
5347 {
5348         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
5349         struct regarray *r;
5350         struct field *fdst, *fidx;
5351         uint32_t dst_struct_id, idx_struct_id, idx_val;
5352
5353         CHECK(n_tokens == 4, EINVAL);
5354
5355         r = regarray_find(p, regarray);
5356         CHECK(r, EINVAL);
5357
5358         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
5359         CHECK(fdst, EINVAL);
5360         CHECK(!fdst->var_size, EINVAL);
5361
5362         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
5363         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5364         if (fidx) {
5365                 CHECK(!fidx->var_size, EINVAL);
5366
5367                 instr->type = INSTR_REGRD_MRM;
5368                 if (dst[0] == 'h' && idx[0] != 'h')
5369                         instr->type = INSTR_REGRD_HRM;
5370                 if (dst[0] != 'h' && idx[0] == 'h')
5371                         instr->type = INSTR_REGRD_MRH;
5372                 if (dst[0] == 'h' && idx[0] == 'h')
5373                         instr->type = INSTR_REGRD_HRH;
5374
5375                 instr->regarray.regarray_id = r->id;
5376                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5377                 instr->regarray.idx.n_bits = fidx->n_bits;
5378                 instr->regarray.idx.offset = fidx->offset / 8;
5379                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5380                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
5381                 instr->regarray.dstsrc.offset = fdst->offset / 8;
5382                 return 0;
5383         }
5384
5385         /* REGRD_MRI, REGRD_HRI. */
5386         idx_val = strtoul(idx, &idx, 0);
5387         CHECK(!idx[0], EINVAL);
5388
5389         instr->type = INSTR_REGRD_MRI;
5390         if (dst[0] == 'h')
5391                 instr->type = INSTR_REGRD_HRI;
5392
5393         instr->regarray.regarray_id = r->id;
5394         instr->regarray.idx_val = idx_val;
5395         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5396         instr->regarray.dstsrc.n_bits = fdst->n_bits;
5397         instr->regarray.dstsrc.offset = fdst->offset / 8;
5398         return 0;
5399 }
5400
5401 static int
5402 instr_regwr_translate(struct rte_swx_pipeline *p,
5403                       struct action *action,
5404                       char **tokens,
5405                       int n_tokens,
5406                       struct instruction *instr,
5407                       struct instruction_data *data __rte_unused)
5408 {
5409         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5410         struct regarray *r;
5411         struct field *fidx, *fsrc;
5412         uint64_t src_val;
5413         uint32_t idx_struct_id, idx_val, src_struct_id;
5414
5415         CHECK(n_tokens == 4, EINVAL);
5416
5417         r = regarray_find(p, regarray);
5418         CHECK(r, EINVAL);
5419
5420         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
5421         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5422         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5423         if (fidx && fsrc) {
5424                 CHECK(!fidx->var_size, EINVAL);
5425                 CHECK(!fsrc->var_size, EINVAL);
5426
5427                 instr->type = INSTR_REGWR_RMM;
5428                 if (idx[0] == 'h' && src[0] != 'h')
5429                         instr->type = INSTR_REGWR_RHM;
5430                 if (idx[0] != 'h' && src[0] == 'h')
5431                         instr->type = INSTR_REGWR_RMH;
5432                 if (idx[0] == 'h' && src[0] == 'h')
5433                         instr->type = INSTR_REGWR_RHH;
5434
5435                 instr->regarray.regarray_id = r->id;
5436                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5437                 instr->regarray.idx.n_bits = fidx->n_bits;
5438                 instr->regarray.idx.offset = fidx->offset / 8;
5439                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5440                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5441                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5442                 return 0;
5443         }
5444
5445         /* REGWR_RHI, REGWR_RMI. */
5446         if (fidx && !fsrc) {
5447                 CHECK(!fidx->var_size, EINVAL);
5448
5449                 src_val = strtoull(src, &src, 0);
5450                 CHECK(!src[0], EINVAL);
5451
5452                 instr->type = INSTR_REGWR_RMI;
5453                 if (idx[0] == 'h')
5454                         instr->type = INSTR_REGWR_RHI;
5455
5456                 instr->regarray.regarray_id = r->id;
5457                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5458                 instr->regarray.idx.n_bits = fidx->n_bits;
5459                 instr->regarray.idx.offset = fidx->offset / 8;
5460                 instr->regarray.dstsrc_val = src_val;
5461                 return 0;
5462         }
5463
5464         /* REGWR_RIH, REGWR_RIM. */
5465         if (!fidx && fsrc) {
5466                 idx_val = strtoul(idx, &idx, 0);
5467                 CHECK(!idx[0], EINVAL);
5468
5469                 CHECK(!fsrc->var_size, EINVAL);
5470
5471                 instr->type = INSTR_REGWR_RIM;
5472                 if (src[0] == 'h')
5473                         instr->type = INSTR_REGWR_RIH;
5474
5475                 instr->regarray.regarray_id = r->id;
5476                 instr->regarray.idx_val = idx_val;
5477                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5478                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5479                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5480                 return 0;
5481         }
5482
5483         /* REGWR_RII. */
5484         src_val = strtoull(src, &src, 0);
5485         CHECK(!src[0], EINVAL);
5486
5487         idx_val = strtoul(idx, &idx, 0);
5488         CHECK(!idx[0], EINVAL);
5489
5490         instr->type = INSTR_REGWR_RII;
5491         instr->regarray.idx_val = idx_val;
5492         instr->regarray.dstsrc_val = src_val;
5493
5494         return 0;
5495 }
5496
5497 static int
5498 instr_regadd_translate(struct rte_swx_pipeline *p,
5499                        struct action *action,
5500                        char **tokens,
5501                        int n_tokens,
5502                        struct instruction *instr,
5503                        struct instruction_data *data __rte_unused)
5504 {
5505         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5506         struct regarray *r;
5507         struct field *fidx, *fsrc;
5508         uint64_t src_val;
5509         uint32_t idx_struct_id, idx_val, src_struct_id;
5510
5511         CHECK(n_tokens == 4, EINVAL);
5512
5513         r = regarray_find(p, regarray);
5514         CHECK(r, EINVAL);
5515
5516         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
5517         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5518         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5519         if (fidx && fsrc) {
5520                 CHECK(!fidx->var_size, EINVAL);
5521                 CHECK(!fsrc->var_size, EINVAL);
5522
5523                 instr->type = INSTR_REGADD_RMM;
5524                 if (idx[0] == 'h' && src[0] != 'h')
5525                         instr->type = INSTR_REGADD_RHM;
5526                 if (idx[0] != 'h' && src[0] == 'h')
5527                         instr->type = INSTR_REGADD_RMH;
5528                 if (idx[0] == 'h' && src[0] == 'h')
5529                         instr->type = INSTR_REGADD_RHH;
5530
5531                 instr->regarray.regarray_id = r->id;
5532                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5533                 instr->regarray.idx.n_bits = fidx->n_bits;
5534                 instr->regarray.idx.offset = fidx->offset / 8;
5535                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5536                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5537                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5538                 return 0;
5539         }
5540
5541         /* REGADD_RHI, REGADD_RMI. */
5542         if (fidx && !fsrc) {
5543                 CHECK(!fidx->var_size, EINVAL);
5544
5545                 src_val = strtoull(src, &src, 0);
5546                 CHECK(!src[0], EINVAL);
5547
5548                 instr->type = INSTR_REGADD_RMI;
5549                 if (idx[0] == 'h')
5550                         instr->type = INSTR_REGADD_RHI;
5551
5552                 instr->regarray.regarray_id = r->id;
5553                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5554                 instr->regarray.idx.n_bits = fidx->n_bits;
5555                 instr->regarray.idx.offset = fidx->offset / 8;
5556                 instr->regarray.dstsrc_val = src_val;
5557                 return 0;
5558         }
5559
5560         /* REGADD_RIH, REGADD_RIM. */
5561         if (!fidx && fsrc) {
5562                 idx_val = strtoul(idx, &idx, 0);
5563                 CHECK(!idx[0], EINVAL);
5564
5565                 CHECK(!fsrc->var_size, EINVAL);
5566
5567                 instr->type = INSTR_REGADD_RIM;
5568                 if (src[0] == 'h')
5569                         instr->type = INSTR_REGADD_RIH;
5570
5571                 instr->regarray.regarray_id = r->id;
5572                 instr->regarray.idx_val = idx_val;
5573                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5574                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5575                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5576                 return 0;
5577         }
5578
5579         /* REGADD_RII. */
5580         src_val = strtoull(src, &src, 0);
5581         CHECK(!src[0], EINVAL);
5582
5583         idx_val = strtoul(idx, &idx, 0);
5584         CHECK(!idx[0], EINVAL);
5585
5586         instr->type = INSTR_REGADD_RII;
5587         instr->regarray.idx_val = idx_val;
5588         instr->regarray.dstsrc_val = src_val;
5589         return 0;
5590 }
5591
5592 static inline uint64_t *
5593 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
5594 {
5595         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5596         return r->regarray;
5597 }
5598
5599 static inline uint64_t
5600 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5601 {
5602         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5603
5604         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5605         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5606         uint64_t idx64 = *idx64_ptr;
5607         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
5608         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5609
5610         return idx;
5611 }
5612
5613 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5614
5615 static inline uint64_t
5616 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5617 {
5618         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5619
5620         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5621         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5622         uint64_t idx64 = *idx64_ptr;
5623         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
5624
5625         return idx;
5626 }
5627
5628 #else
5629
5630 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
5631
5632 #endif
5633
5634 static inline uint64_t
5635 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5636 {
5637         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5638
5639         uint64_t idx = ip->regarray.idx_val & r->size_mask;
5640
5641         return idx;
5642 }
5643
5644 static inline uint64_t
5645 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
5646 {
5647         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5648         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5649         uint64_t src64 = *src64_ptr;
5650         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5651         uint64_t src = src64 & src64_mask;
5652
5653         return src;
5654 }
5655
5656 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5657
5658 static inline uint64_t
5659 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
5660 {
5661         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5662         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5663         uint64_t src64 = *src64_ptr;
5664         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
5665
5666         return src;
5667 }
5668
5669 #else
5670
5671 #define instr_regarray_src_nbo instr_regarray_src_hbo
5672
5673 #endif
5674
5675 static inline void
5676 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5677 {
5678         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5679         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5680         uint64_t dst64 = *dst64_ptr;
5681         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5682
5683         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5684
5685 }
5686
5687 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5688
5689 static inline void
5690 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5691 {
5692         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5693         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5694         uint64_t dst64 = *dst64_ptr;
5695         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5696
5697         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
5698         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5699 }
5700
5701 #else
5702
5703 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
5704
5705 #endif
5706
5707 static inline void
5708 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
5709 {
5710         struct thread *t = &p->threads[p->thread_id];
5711         struct instruction *ip = t->ip;
5712         uint64_t *regarray, idx;
5713
5714         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
5715
5716         /* Structs. */
5717         regarray = instr_regarray_regarray(p, ip);
5718         idx = instr_regarray_idx_nbo(p, t, ip);
5719         rte_prefetch0(&regarray[idx]);
5720
5721         /* Thread. */
5722         thread_ip_inc(p);
5723 }
5724
5725 static inline void
5726 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
5727 {
5728         struct thread *t = &p->threads[p->thread_id];
5729         struct instruction *ip = t->ip;
5730         uint64_t *regarray, idx;
5731
5732         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
5733
5734         /* Structs. */
5735         regarray = instr_regarray_regarray(p, ip);
5736         idx = instr_regarray_idx_hbo(p, t, ip);
5737         rte_prefetch0(&regarray[idx]);
5738
5739         /* Thread. */
5740         thread_ip_inc(p);
5741 }
5742
5743 static inline void
5744 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
5745 {
5746         struct thread *t = &p->threads[p->thread_id];
5747         struct instruction *ip = t->ip;
5748         uint64_t *regarray, idx;
5749
5750         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
5751
5752         /* Structs. */
5753         regarray = instr_regarray_regarray(p, ip);
5754         idx = instr_regarray_idx_imm(p, ip);
5755         rte_prefetch0(&regarray[idx]);
5756
5757         /* Thread. */
5758         thread_ip_inc(p);
5759 }
5760
5761 static inline void
5762 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
5763 {
5764         struct thread *t = &p->threads[p->thread_id];
5765         struct instruction *ip = t->ip;
5766         uint64_t *regarray, idx;
5767
5768         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
5769
5770         /* Structs. */
5771         regarray = instr_regarray_regarray(p, ip);
5772         idx = instr_regarray_idx_nbo(p, t, ip);
5773         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5774
5775         /* Thread. */
5776         thread_ip_inc(p);
5777 }
5778
5779 static inline void
5780 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
5781 {
5782         struct thread *t = &p->threads[p->thread_id];
5783         struct instruction *ip = t->ip;
5784         uint64_t *regarray, idx;
5785
5786         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
5787
5788         /* Structs. */
5789         regarray = instr_regarray_regarray(p, ip);
5790         idx = instr_regarray_idx_hbo(p, t, ip);
5791         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5792
5793         /* Thread. */
5794         thread_ip_inc(p);
5795 }
5796
5797 static inline void
5798 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
5799 {
5800         struct thread *t = &p->threads[p->thread_id];
5801         struct instruction *ip = t->ip;
5802         uint64_t *regarray, idx;
5803
5804         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
5805
5806         /* Structs. */
5807         regarray = instr_regarray_regarray(p, ip);
5808         idx = instr_regarray_idx_nbo(p, t, ip);
5809         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5810
5811         /* Thread. */
5812         thread_ip_inc(p);
5813 }
5814
5815 static inline void
5816 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
5817 {
5818         struct thread *t = &p->threads[p->thread_id];
5819         struct instruction *ip = t->ip;
5820         uint64_t *regarray, idx;
5821
5822         /* Structs. */
5823         regarray = instr_regarray_regarray(p, ip);
5824         idx = instr_regarray_idx_hbo(p, t, ip);
5825         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5826
5827         /* Thread. */
5828         thread_ip_inc(p);
5829 }
5830
5831 static inline void
5832 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
5833 {
5834         struct thread *t = &p->threads[p->thread_id];
5835         struct instruction *ip = t->ip;
5836         uint64_t *regarray, idx;
5837
5838         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
5839
5840         /* Structs. */
5841         regarray = instr_regarray_regarray(p, ip);
5842         idx = instr_regarray_idx_imm(p, ip);
5843         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5844
5845         /* Thread. */
5846         thread_ip_inc(p);
5847 }
5848
5849 static inline void
5850 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
5851 {
5852         struct thread *t = &p->threads[p->thread_id];
5853         struct instruction *ip = t->ip;
5854         uint64_t *regarray, idx;
5855
5856         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
5857
5858         /* Structs. */
5859         regarray = instr_regarray_regarray(p, ip);
5860         idx = instr_regarray_idx_imm(p, ip);
5861         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5862
5863         /* Thread. */
5864         thread_ip_inc(p);
5865 }
5866
5867 static inline void
5868 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
5869 {
5870         struct thread *t = &p->threads[p->thread_id];
5871         struct instruction *ip = t->ip;
5872         uint64_t *regarray, idx, src;
5873
5874         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
5875
5876         /* Structs. */
5877         regarray = instr_regarray_regarray(p, ip);
5878         idx = instr_regarray_idx_nbo(p, t, ip);
5879         src = instr_regarray_src_nbo(t, ip);
5880         regarray[idx] = src;
5881
5882         /* Thread. */
5883         thread_ip_inc(p);
5884 }
5885
5886 static inline void
5887 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
5888 {
5889         struct thread *t = &p->threads[p->thread_id];
5890         struct instruction *ip = t->ip;
5891         uint64_t *regarray, idx, src;
5892
5893         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
5894
5895         /* Structs. */
5896         regarray = instr_regarray_regarray(p, ip);
5897         idx = instr_regarray_idx_nbo(p, t, ip);
5898         src = instr_regarray_src_hbo(t, ip);
5899         regarray[idx] = src;
5900
5901         /* Thread. */
5902         thread_ip_inc(p);
5903 }
5904
5905 static inline void
5906 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
5907 {
5908         struct thread *t = &p->threads[p->thread_id];
5909         struct instruction *ip = t->ip;
5910         uint64_t *regarray, idx, src;
5911
5912         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
5913
5914         /* Structs. */
5915         regarray = instr_regarray_regarray(p, ip);
5916         idx = instr_regarray_idx_hbo(p, t, ip);
5917         src = instr_regarray_src_nbo(t, ip);
5918         regarray[idx] = src;
5919
5920         /* Thread. */
5921         thread_ip_inc(p);
5922 }
5923
5924 static inline void
5925 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
5926 {
5927         struct thread *t = &p->threads[p->thread_id];
5928         struct instruction *ip = t->ip;
5929         uint64_t *regarray, idx, src;
5930
5931         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
5932
5933         /* Structs. */
5934         regarray = instr_regarray_regarray(p, ip);
5935         idx = instr_regarray_idx_hbo(p, t, ip);
5936         src = instr_regarray_src_hbo(t, ip);
5937         regarray[idx] = src;
5938
5939         /* Thread. */
5940         thread_ip_inc(p);
5941 }
5942
5943 static inline void
5944 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
5945 {
5946         struct thread *t = &p->threads[p->thread_id];
5947         struct instruction *ip = t->ip;
5948         uint64_t *regarray, idx, src;
5949
5950         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
5951
5952         /* Structs. */
5953         regarray = instr_regarray_regarray(p, ip);
5954         idx = instr_regarray_idx_nbo(p, t, ip);
5955         src = ip->regarray.dstsrc_val;
5956         regarray[idx] = src;
5957
5958         /* Thread. */
5959         thread_ip_inc(p);
5960 }
5961
5962 static inline void
5963 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
5964 {
5965         struct thread *t = &p->threads[p->thread_id];
5966         struct instruction *ip = t->ip;
5967         uint64_t *regarray, idx, src;
5968
5969         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
5970
5971         /* Structs. */
5972         regarray = instr_regarray_regarray(p, ip);
5973         idx = instr_regarray_idx_hbo(p, t, ip);
5974         src = ip->regarray.dstsrc_val;
5975         regarray[idx] = src;
5976
5977         /* Thread. */
5978         thread_ip_inc(p);
5979 }
5980
5981 static inline void
5982 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
5983 {
5984         struct thread *t = &p->threads[p->thread_id];
5985         struct instruction *ip = t->ip;
5986         uint64_t *regarray, idx, src;
5987
5988         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
5989
5990         /* Structs. */
5991         regarray = instr_regarray_regarray(p, ip);
5992         idx = instr_regarray_idx_imm(p, ip);
5993         src = instr_regarray_src_nbo(t, ip);
5994         regarray[idx] = src;
5995
5996         /* Thread. */
5997         thread_ip_inc(p);
5998 }
5999
6000 static inline void
6001 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
6002 {
6003         struct thread *t = &p->threads[p->thread_id];
6004         struct instruction *ip = t->ip;
6005         uint64_t *regarray, idx, src;
6006
6007         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
6008
6009         /* Structs. */
6010         regarray = instr_regarray_regarray(p, ip);
6011         idx = instr_regarray_idx_imm(p, ip);
6012         src = instr_regarray_src_hbo(t, ip);
6013         regarray[idx] = src;
6014
6015         /* Thread. */
6016         thread_ip_inc(p);
6017 }
6018
6019 static inline void
6020 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
6021 {
6022         struct thread *t = &p->threads[p->thread_id];
6023         struct instruction *ip = t->ip;
6024         uint64_t *regarray, idx, src;
6025
6026         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
6027
6028         /* Structs. */
6029         regarray = instr_regarray_regarray(p, ip);
6030         idx = instr_regarray_idx_imm(p, ip);
6031         src = ip->regarray.dstsrc_val;
6032         regarray[idx] = src;
6033
6034         /* Thread. */
6035         thread_ip_inc(p);
6036 }
6037
6038 static inline void
6039 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
6040 {
6041         struct thread *t = &p->threads[p->thread_id];
6042         struct instruction *ip = t->ip;
6043         uint64_t *regarray, idx, src;
6044
6045         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
6046
6047         /* Structs. */
6048         regarray = instr_regarray_regarray(p, ip);
6049         idx = instr_regarray_idx_nbo(p, t, ip);
6050         src = instr_regarray_src_nbo(t, ip);
6051         regarray[idx] += src;
6052
6053         /* Thread. */
6054         thread_ip_inc(p);
6055 }
6056
6057 static inline void
6058 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
6059 {
6060         struct thread *t = &p->threads[p->thread_id];
6061         struct instruction *ip = t->ip;
6062         uint64_t *regarray, idx, src;
6063
6064         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
6065
6066         /* Structs. */
6067         regarray = instr_regarray_regarray(p, ip);
6068         idx = instr_regarray_idx_nbo(p, t, ip);
6069         src = instr_regarray_src_hbo(t, ip);
6070         regarray[idx] += src;
6071
6072         /* Thread. */
6073         thread_ip_inc(p);
6074 }
6075
6076 static inline void
6077 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
6078 {
6079         struct thread *t = &p->threads[p->thread_id];
6080         struct instruction *ip = t->ip;
6081         uint64_t *regarray, idx, src;
6082
6083         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
6084
6085         /* Structs. */
6086         regarray = instr_regarray_regarray(p, ip);
6087         idx = instr_regarray_idx_hbo(p, t, ip);
6088         src = instr_regarray_src_nbo(t, ip);
6089         regarray[idx] += src;
6090
6091         /* Thread. */
6092         thread_ip_inc(p);
6093 }
6094
6095 static inline void
6096 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
6097 {
6098         struct thread *t = &p->threads[p->thread_id];
6099         struct instruction *ip = t->ip;
6100         uint64_t *regarray, idx, src;
6101
6102         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
6103
6104         /* Structs. */
6105         regarray = instr_regarray_regarray(p, ip);
6106         idx = instr_regarray_idx_hbo(p, t, ip);
6107         src = instr_regarray_src_hbo(t, ip);
6108         regarray[idx] += src;
6109
6110         /* Thread. */
6111         thread_ip_inc(p);
6112 }
6113
6114 static inline void
6115 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
6116 {
6117         struct thread *t = &p->threads[p->thread_id];
6118         struct instruction *ip = t->ip;
6119         uint64_t *regarray, idx, src;
6120
6121         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
6122
6123         /* Structs. */
6124         regarray = instr_regarray_regarray(p, ip);
6125         idx = instr_regarray_idx_nbo(p, t, ip);
6126         src = ip->regarray.dstsrc_val;
6127         regarray[idx] += src;
6128
6129         /* Thread. */
6130         thread_ip_inc(p);
6131 }
6132
6133 static inline void
6134 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
6135 {
6136         struct thread *t = &p->threads[p->thread_id];
6137         struct instruction *ip = t->ip;
6138         uint64_t *regarray, idx, src;
6139
6140         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
6141
6142         /* Structs. */
6143         regarray = instr_regarray_regarray(p, ip);
6144         idx = instr_regarray_idx_hbo(p, t, ip);
6145         src = ip->regarray.dstsrc_val;
6146         regarray[idx] += src;
6147
6148         /* Thread. */
6149         thread_ip_inc(p);
6150 }
6151
6152 static inline void
6153 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
6154 {
6155         struct thread *t = &p->threads[p->thread_id];
6156         struct instruction *ip = t->ip;
6157         uint64_t *regarray, idx, src;
6158
6159         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
6160
6161         /* Structs. */
6162         regarray = instr_regarray_regarray(p, ip);
6163         idx = instr_regarray_idx_imm(p, ip);
6164         src = instr_regarray_src_nbo(t, ip);
6165         regarray[idx] += src;
6166
6167         /* Thread. */
6168         thread_ip_inc(p);
6169 }
6170
6171 static inline void
6172 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
6173 {
6174         struct thread *t = &p->threads[p->thread_id];
6175         struct instruction *ip = t->ip;
6176         uint64_t *regarray, idx, src;
6177
6178         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
6179
6180         /* Structs. */
6181         regarray = instr_regarray_regarray(p, ip);
6182         idx = instr_regarray_idx_imm(p, ip);
6183         src = instr_regarray_src_hbo(t, ip);
6184         regarray[idx] += src;
6185
6186         /* Thread. */
6187         thread_ip_inc(p);
6188 }
6189
6190 static inline void
6191 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
6192 {
6193         struct thread *t = &p->threads[p->thread_id];
6194         struct instruction *ip = t->ip;
6195         uint64_t *regarray, idx, src;
6196
6197         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
6198
6199         /* Structs. */
6200         regarray = instr_regarray_regarray(p, ip);
6201         idx = instr_regarray_idx_imm(p, ip);
6202         src = ip->regarray.dstsrc_val;
6203         regarray[idx] += src;
6204
6205         /* Thread. */
6206         thread_ip_inc(p);
6207 }
6208
6209 /*
6210  * metarray.
6211  */
6212 static struct metarray *
6213 metarray_find(struct rte_swx_pipeline *p, const char *name);
6214
6215 static int
6216 instr_metprefetch_translate(struct rte_swx_pipeline *p,
6217                             struct action *action,
6218                             char **tokens,
6219                             int n_tokens,
6220                             struct instruction *instr,
6221                             struct instruction_data *data __rte_unused)
6222 {
6223         char *metarray = tokens[1], *idx = tokens[2];
6224         struct metarray *m;
6225         struct field *fidx;
6226         uint32_t idx_struct_id, idx_val;
6227
6228         CHECK(n_tokens == 3, EINVAL);
6229
6230         m = metarray_find(p, metarray);
6231         CHECK(m, EINVAL);
6232
6233         /* METPREFETCH_H, METPREFETCH_M. */
6234         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6235         if (fidx) {
6236                 CHECK(!fidx->var_size, EINVAL);
6237
6238                 instr->type = INSTR_METPREFETCH_M;
6239                 if (idx[0] == 'h')
6240                         instr->type = INSTR_METPREFETCH_H;
6241
6242                 instr->meter.metarray_id = m->id;
6243                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6244                 instr->meter.idx.n_bits = fidx->n_bits;
6245                 instr->meter.idx.offset = fidx->offset / 8;
6246                 return 0;
6247         }
6248
6249         /* METPREFETCH_I. */
6250         idx_val = strtoul(idx, &idx, 0);
6251         CHECK(!idx[0], EINVAL);
6252
6253         instr->type = INSTR_METPREFETCH_I;
6254         instr->meter.metarray_id = m->id;
6255         instr->meter.idx_val = idx_val;
6256         return 0;
6257 }
6258
6259 static int
6260 instr_meter_translate(struct rte_swx_pipeline *p,
6261                       struct action *action,
6262                       char **tokens,
6263                       int n_tokens,
6264                       struct instruction *instr,
6265                       struct instruction_data *data __rte_unused)
6266 {
6267         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
6268         char *color_in = tokens[4], *color_out = tokens[5];
6269         struct metarray *m;
6270         struct field *fidx, *flength, *fcin, *fcout;
6271         uint32_t idx_struct_id, length_struct_id;
6272         uint32_t color_in_struct_id, color_out_struct_id;
6273
6274         CHECK(n_tokens == 6, EINVAL);
6275
6276         m = metarray_find(p, metarray);
6277         CHECK(m, EINVAL);
6278
6279         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6280
6281         flength = struct_field_parse(p, action, length, &length_struct_id);
6282         CHECK(flength, EINVAL);
6283         CHECK(!flength->var_size, EINVAL);
6284
6285         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
6286
6287         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
6288         CHECK(fcout, EINVAL);
6289         CHECK(!fcout->var_size, EINVAL);
6290
6291         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
6292         if (fidx && fcin) {
6293                 CHECK(!fidx->var_size, EINVAL);
6294                 CHECK(!fcin->var_size, EINVAL);
6295
6296                 instr->type = INSTR_METER_MMM;
6297                 if (idx[0] == 'h' && length[0] == 'h')
6298                         instr->type = INSTR_METER_HHM;
6299                 if (idx[0] == 'h' && length[0] != 'h')
6300                         instr->type = INSTR_METER_HMM;
6301                 if (idx[0] != 'h' && length[0] == 'h')
6302                         instr->type = INSTR_METER_MHM;
6303
6304                 instr->meter.metarray_id = m->id;
6305
6306                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6307                 instr->meter.idx.n_bits = fidx->n_bits;
6308                 instr->meter.idx.offset = fidx->offset / 8;
6309
6310                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6311                 instr->meter.length.n_bits = flength->n_bits;
6312                 instr->meter.length.offset = flength->offset / 8;
6313
6314                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6315                 instr->meter.color_in.n_bits = fcin->n_bits;
6316                 instr->meter.color_in.offset = fcin->offset / 8;
6317
6318                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6319                 instr->meter.color_out.n_bits = fcout->n_bits;
6320                 instr->meter.color_out.offset = fcout->offset / 8;
6321
6322                 return 0;
6323         }
6324
6325         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
6326         if (fidx && !fcin) {
6327                 uint32_t color_in_val;
6328
6329                 CHECK(!fidx->var_size, EINVAL);
6330
6331                 color_in_val = strtoul(color_in, &color_in, 0);
6332                 CHECK(!color_in[0], EINVAL);
6333
6334                 instr->type = INSTR_METER_MMI;
6335                 if (idx[0] == 'h' && length[0] == 'h')
6336                         instr->type = INSTR_METER_HHI;
6337                 if (idx[0] == 'h' && length[0] != 'h')
6338                         instr->type = INSTR_METER_HMI;
6339                 if (idx[0] != 'h' && length[0] == 'h')
6340                         instr->type = INSTR_METER_MHI;
6341
6342                 instr->meter.metarray_id = m->id;
6343
6344                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6345                 instr->meter.idx.n_bits = fidx->n_bits;
6346                 instr->meter.idx.offset = fidx->offset / 8;
6347
6348                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6349                 instr->meter.length.n_bits = flength->n_bits;
6350                 instr->meter.length.offset = flength->offset / 8;
6351
6352                 instr->meter.color_in_val = color_in_val;
6353
6354                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6355                 instr->meter.color_out.n_bits = fcout->n_bits;
6356                 instr->meter.color_out.offset = fcout->offset / 8;
6357
6358                 return 0;
6359         }
6360
6361         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
6362         if (!fidx && fcin) {
6363                 uint32_t idx_val;
6364
6365                 idx_val = strtoul(idx, &idx, 0);
6366                 CHECK(!idx[0], EINVAL);
6367
6368                 CHECK(!fcin->var_size, EINVAL);
6369
6370                 instr->type = INSTR_METER_IMM;
6371                 if (length[0] == 'h')
6372                         instr->type = INSTR_METER_IHM;
6373
6374                 instr->meter.metarray_id = m->id;
6375
6376                 instr->meter.idx_val = idx_val;
6377
6378                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6379                 instr->meter.length.n_bits = flength->n_bits;
6380                 instr->meter.length.offset = flength->offset / 8;
6381
6382                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6383                 instr->meter.color_in.n_bits = fcin->n_bits;
6384                 instr->meter.color_in.offset = fcin->offset / 8;
6385
6386                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6387                 instr->meter.color_out.n_bits = fcout->n_bits;
6388                 instr->meter.color_out.offset = fcout->offset / 8;
6389
6390                 return 0;
6391         }
6392
6393         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
6394         if (!fidx && !fcin) {
6395                 uint32_t idx_val, color_in_val;
6396
6397                 idx_val = strtoul(idx, &idx, 0);
6398                 CHECK(!idx[0], EINVAL);
6399
6400                 color_in_val = strtoul(color_in, &color_in, 0);
6401                 CHECK(!color_in[0], EINVAL);
6402
6403                 instr->type = INSTR_METER_IMI;
6404                 if (length[0] == 'h')
6405                         instr->type = INSTR_METER_IHI;
6406
6407                 instr->meter.metarray_id = m->id;
6408
6409                 instr->meter.idx_val = idx_val;
6410
6411                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6412                 instr->meter.length.n_bits = flength->n_bits;
6413                 instr->meter.length.offset = flength->offset / 8;
6414
6415                 instr->meter.color_in_val = color_in_val;
6416
6417                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6418                 instr->meter.color_out.n_bits = fcout->n_bits;
6419                 instr->meter.color_out.offset = fcout->offset / 8;
6420
6421                 return 0;
6422         }
6423
6424         CHECK(0, EINVAL);
6425 }
6426
6427 static inline struct meter *
6428 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6429 {
6430         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6431
6432         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6433         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6434         uint64_t idx64 = *idx64_ptr;
6435         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
6436         uint64_t idx = idx64 & idx64_mask & r->size_mask;
6437
6438         return &r->metarray[idx];
6439 }
6440
6441 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6442
6443 static inline struct meter *
6444 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6445 {
6446         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6447
6448         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6449         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6450         uint64_t idx64 = *idx64_ptr;
6451         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
6452
6453         return &r->metarray[idx];
6454 }
6455
6456 #else
6457
6458 #define instr_meter_idx_nbo instr_meter_idx_hbo
6459
6460 #endif
6461
6462 static inline struct meter *
6463 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
6464 {
6465         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6466
6467         uint64_t idx =  ip->meter.idx_val & r->size_mask;
6468
6469         return &r->metarray[idx];
6470 }
6471
6472 static inline uint32_t
6473 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
6474 {
6475         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6476         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6477         uint64_t src64 = *src64_ptr;
6478         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
6479         uint64_t src = src64 & src64_mask;
6480
6481         return (uint32_t)src;
6482 }
6483
6484 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6485
6486 static inline uint32_t
6487 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
6488 {
6489         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6490         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6491         uint64_t src64 = *src64_ptr;
6492         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
6493
6494         return (uint32_t)src;
6495 }
6496
6497 #else
6498
6499 #define instr_meter_length_nbo instr_meter_length_hbo
6500
6501 #endif
6502
6503 static inline enum rte_color
6504 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
6505 {
6506         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
6507         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
6508         uint64_t src64 = *src64_ptr;
6509         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
6510         uint64_t src = src64 & src64_mask;
6511
6512         return (enum rte_color)src;
6513 }
6514
6515 static inline void
6516 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
6517 {
6518         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
6519         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
6520         uint64_t dst64 = *dst64_ptr;
6521         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
6522
6523         uint64_t src = (uint64_t)color_out;
6524
6525         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6526 }
6527
6528 static inline void
6529 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
6530 {
6531         struct thread *t = &p->threads[p->thread_id];
6532         struct instruction *ip = t->ip;
6533         struct meter *m;
6534
6535         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
6536
6537         /* Structs. */
6538         m = instr_meter_idx_nbo(p, t, ip);
6539         rte_prefetch0(m);
6540
6541         /* Thread. */
6542         thread_ip_inc(p);
6543 }
6544
6545 static inline void
6546 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
6547 {
6548         struct thread *t = &p->threads[p->thread_id];
6549         struct instruction *ip = t->ip;
6550         struct meter *m;
6551
6552         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
6553
6554         /* Structs. */
6555         m = instr_meter_idx_hbo(p, t, ip);
6556         rte_prefetch0(m);
6557
6558         /* Thread. */
6559         thread_ip_inc(p);
6560 }
6561
6562 static inline void
6563 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
6564 {
6565         struct thread *t = &p->threads[p->thread_id];
6566         struct instruction *ip = t->ip;
6567         struct meter *m;
6568
6569         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
6570
6571         /* Structs. */
6572         m = instr_meter_idx_imm(p, ip);
6573         rte_prefetch0(m);
6574
6575         /* Thread. */
6576         thread_ip_inc(p);
6577 }
6578
6579 static inline void
6580 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
6581 {
6582         struct thread *t = &p->threads[p->thread_id];
6583         struct instruction *ip = t->ip;
6584         struct meter *m;
6585         uint64_t time, n_pkts, n_bytes;
6586         uint32_t length;
6587         enum rte_color color_in, color_out;
6588
6589         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
6590
6591         /* Structs. */
6592         m = instr_meter_idx_nbo(p, t, ip);
6593         rte_prefetch0(m->n_pkts);
6594         time = rte_get_tsc_cycles();
6595         length = instr_meter_length_nbo(t, ip);
6596         color_in = instr_meter_color_in_hbo(t, ip);
6597
6598         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6599                 &m->profile->profile,
6600                 time,
6601                 length,
6602                 color_in);
6603
6604         color_out &= m->color_mask;
6605
6606         n_pkts = m->n_pkts[color_out];
6607         n_bytes = m->n_bytes[color_out];
6608
6609         instr_meter_color_out_hbo_set(t, ip, color_out);
6610
6611         m->n_pkts[color_out] = n_pkts + 1;
6612         m->n_bytes[color_out] = n_bytes + length;
6613
6614         /* Thread. */
6615         thread_ip_inc(p);
6616 }
6617
6618 static inline void
6619 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
6620 {
6621         struct thread *t = &p->threads[p->thread_id];
6622         struct instruction *ip = t->ip;
6623         struct meter *m;
6624         uint64_t time, n_pkts, n_bytes;
6625         uint32_t length;
6626         enum rte_color color_in, color_out;
6627
6628         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
6629
6630         /* Structs. */
6631         m = instr_meter_idx_nbo(p, t, ip);
6632         rte_prefetch0(m->n_pkts);
6633         time = rte_get_tsc_cycles();
6634         length = instr_meter_length_nbo(t, ip);
6635         color_in = (enum rte_color)ip->meter.color_in_val;
6636
6637         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6638                 &m->profile->profile,
6639                 time,
6640                 length,
6641                 color_in);
6642
6643         color_out &= m->color_mask;
6644
6645         n_pkts = m->n_pkts[color_out];
6646         n_bytes = m->n_bytes[color_out];
6647
6648         instr_meter_color_out_hbo_set(t, ip, color_out);
6649
6650         m->n_pkts[color_out] = n_pkts + 1;
6651         m->n_bytes[color_out] = n_bytes + length;
6652
6653         /* Thread. */
6654         thread_ip_inc(p);
6655 }
6656
6657 static inline void
6658 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
6659 {
6660         struct thread *t = &p->threads[p->thread_id];
6661         struct instruction *ip = t->ip;
6662         struct meter *m;
6663         uint64_t time, n_pkts, n_bytes;
6664         uint32_t length;
6665         enum rte_color color_in, color_out;
6666
6667         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
6668
6669         /* Structs. */
6670         m = instr_meter_idx_nbo(p, t, ip);
6671         rte_prefetch0(m->n_pkts);
6672         time = rte_get_tsc_cycles();
6673         length = instr_meter_length_hbo(t, ip);
6674         color_in = instr_meter_color_in_hbo(t, ip);
6675
6676         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6677                 &m->profile->profile,
6678                 time,
6679                 length,
6680                 color_in);
6681
6682         color_out &= m->color_mask;
6683
6684         n_pkts = m->n_pkts[color_out];
6685         n_bytes = m->n_bytes[color_out];
6686
6687         instr_meter_color_out_hbo_set(t, ip, color_out);
6688
6689         m->n_pkts[color_out] = n_pkts + 1;
6690         m->n_bytes[color_out] = n_bytes + length;
6691
6692         /* Thread. */
6693         thread_ip_inc(p);
6694 }
6695 static inline void
6696 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
6697 {
6698         struct thread *t = &p->threads[p->thread_id];
6699         struct instruction *ip = t->ip;
6700         struct meter *m;
6701         uint64_t time, n_pkts, n_bytes;
6702         uint32_t length;
6703         enum rte_color color_in, color_out;
6704
6705         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
6706
6707         /* Structs. */
6708         m = instr_meter_idx_nbo(p, t, ip);
6709         rte_prefetch0(m->n_pkts);
6710         time = rte_get_tsc_cycles();
6711         length = instr_meter_length_hbo(t, ip);
6712         color_in = (enum rte_color)ip->meter.color_in_val;
6713
6714         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6715                 &m->profile->profile,
6716                 time,
6717                 length,
6718                 color_in);
6719
6720         color_out &= m->color_mask;
6721
6722         n_pkts = m->n_pkts[color_out];
6723         n_bytes = m->n_bytes[color_out];
6724
6725         instr_meter_color_out_hbo_set(t, ip, color_out);
6726
6727         m->n_pkts[color_out] = n_pkts + 1;
6728         m->n_bytes[color_out] = n_bytes + length;
6729
6730         /* Thread. */
6731         thread_ip_inc(p);
6732 }
6733
6734 static inline void
6735 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
6736 {
6737         struct thread *t = &p->threads[p->thread_id];
6738         struct instruction *ip = t->ip;
6739         struct meter *m;
6740         uint64_t time, n_pkts, n_bytes;
6741         uint32_t length;
6742         enum rte_color color_in, color_out;
6743
6744         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
6745
6746         /* Structs. */
6747         m = instr_meter_idx_hbo(p, t, ip);
6748         rte_prefetch0(m->n_pkts);
6749         time = rte_get_tsc_cycles();
6750         length = instr_meter_length_nbo(t, ip);
6751         color_in = instr_meter_color_in_hbo(t, ip);
6752
6753         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6754                 &m->profile->profile,
6755                 time,
6756                 length,
6757                 color_in);
6758
6759         color_out &= m->color_mask;
6760
6761         n_pkts = m->n_pkts[color_out];
6762         n_bytes = m->n_bytes[color_out];
6763
6764         instr_meter_color_out_hbo_set(t, ip, color_out);
6765
6766         m->n_pkts[color_out] = n_pkts + 1;
6767         m->n_bytes[color_out] = n_bytes + length;
6768
6769         /* Thread. */
6770         thread_ip_inc(p);
6771 }
6772
6773 static inline void
6774 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
6775 {
6776         struct thread *t = &p->threads[p->thread_id];
6777         struct instruction *ip = t->ip;
6778         struct meter *m;
6779         uint64_t time, n_pkts, n_bytes;
6780         uint32_t length;
6781         enum rte_color color_in, color_out;
6782
6783         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
6784
6785         /* Structs. */
6786         m = instr_meter_idx_hbo(p, t, ip);
6787         rte_prefetch0(m->n_pkts);
6788         time = rte_get_tsc_cycles();
6789         length = instr_meter_length_nbo(t, ip);
6790         color_in = (enum rte_color)ip->meter.color_in_val;
6791
6792         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6793                 &m->profile->profile,
6794                 time,
6795                 length,
6796                 color_in);
6797
6798         color_out &= m->color_mask;
6799
6800         n_pkts = m->n_pkts[color_out];
6801         n_bytes = m->n_bytes[color_out];
6802
6803         instr_meter_color_out_hbo_set(t, ip, color_out);
6804
6805         m->n_pkts[color_out] = n_pkts + 1;
6806         m->n_bytes[color_out] = n_bytes + length;
6807
6808         /* Thread. */
6809         thread_ip_inc(p);
6810 }
6811
6812 static inline void
6813 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
6814 {
6815         struct thread *t = &p->threads[p->thread_id];
6816         struct instruction *ip = t->ip;
6817         struct meter *m;
6818         uint64_t time, n_pkts, n_bytes;
6819         uint32_t length;
6820         enum rte_color color_in, color_out;
6821
6822         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
6823
6824         /* Structs. */
6825         m = instr_meter_idx_hbo(p, t, ip);
6826         rte_prefetch0(m->n_pkts);
6827         time = rte_get_tsc_cycles();
6828         length = instr_meter_length_hbo(t, ip);
6829         color_in = instr_meter_color_in_hbo(t, ip);
6830
6831         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6832                 &m->profile->profile,
6833                 time,
6834                 length,
6835                 color_in);
6836
6837         color_out &= m->color_mask;
6838
6839         n_pkts = m->n_pkts[color_out];
6840         n_bytes = m->n_bytes[color_out];
6841
6842         instr_meter_color_out_hbo_set(t, ip, color_out);
6843
6844         m->n_pkts[color_out] = n_pkts + 1;
6845         m->n_bytes[color_out] = n_bytes + length;
6846
6847         /* Thread. */
6848         thread_ip_inc(p);
6849 }
6850
6851 static inline void
6852 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
6853 {
6854         struct thread *t = &p->threads[p->thread_id];
6855         struct instruction *ip = t->ip;
6856         struct meter *m;
6857         uint64_t time, n_pkts, n_bytes;
6858         uint32_t length;
6859         enum rte_color color_in, color_out;
6860
6861         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
6862
6863         /* Structs. */
6864         m = instr_meter_idx_hbo(p, t, ip);
6865         rte_prefetch0(m->n_pkts);
6866         time = rte_get_tsc_cycles();
6867         length = instr_meter_length_hbo(t, ip);
6868         color_in = (enum rte_color)ip->meter.color_in_val;
6869
6870         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6871                 &m->profile->profile,
6872                 time,
6873                 length,
6874                 color_in);
6875
6876         color_out &= m->color_mask;
6877
6878         n_pkts = m->n_pkts[color_out];
6879         n_bytes = m->n_bytes[color_out];
6880
6881         instr_meter_color_out_hbo_set(t, ip, color_out);
6882
6883         m->n_pkts[color_out] = n_pkts + 1;
6884         m->n_bytes[color_out] = n_bytes + length;
6885
6886         /* Thread. */
6887         thread_ip_inc(p);
6888 }
6889
6890 static inline void
6891 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
6892 {
6893         struct thread *t = &p->threads[p->thread_id];
6894         struct instruction *ip = t->ip;
6895         struct meter *m;
6896         uint64_t time, n_pkts, n_bytes;
6897         uint32_t length;
6898         enum rte_color color_in, color_out;
6899
6900         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
6901
6902         /* Structs. */
6903         m = instr_meter_idx_imm(p, ip);
6904         rte_prefetch0(m->n_pkts);
6905         time = rte_get_tsc_cycles();
6906         length = instr_meter_length_nbo(t, ip);
6907         color_in = instr_meter_color_in_hbo(t, ip);
6908
6909         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6910                 &m->profile->profile,
6911                 time,
6912                 length,
6913                 color_in);
6914
6915         color_out &= m->color_mask;
6916
6917         n_pkts = m->n_pkts[color_out];
6918         n_bytes = m->n_bytes[color_out];
6919
6920         instr_meter_color_out_hbo_set(t, ip, color_out);
6921
6922         m->n_pkts[color_out] = n_pkts + 1;
6923         m->n_bytes[color_out] = n_bytes + length;
6924
6925         /* Thread. */
6926         thread_ip_inc(p);
6927 }
6928
6929 static inline void
6930 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
6931 {
6932         struct thread *t = &p->threads[p->thread_id];
6933         struct instruction *ip = t->ip;
6934         struct meter *m;
6935         uint64_t time, n_pkts, n_bytes;
6936         uint32_t length;
6937         enum rte_color color_in, color_out;
6938
6939         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
6940
6941         /* Structs. */
6942         m = instr_meter_idx_imm(p, ip);
6943         rte_prefetch0(m->n_pkts);
6944         time = rte_get_tsc_cycles();
6945         length = instr_meter_length_nbo(t, ip);
6946         color_in = (enum rte_color)ip->meter.color_in_val;
6947
6948         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6949                 &m->profile->profile,
6950                 time,
6951                 length,
6952                 color_in);
6953
6954         color_out &= m->color_mask;
6955
6956         n_pkts = m->n_pkts[color_out];
6957         n_bytes = m->n_bytes[color_out];
6958
6959         instr_meter_color_out_hbo_set(t, ip, color_out);
6960
6961         m->n_pkts[color_out] = n_pkts + 1;
6962         m->n_bytes[color_out] = n_bytes + length;
6963
6964         /* Thread. */
6965         thread_ip_inc(p);
6966 }
6967
6968 static inline void
6969 instr_meter_imm_exec(struct rte_swx_pipeline *p)
6970 {
6971         struct thread *t = &p->threads[p->thread_id];
6972         struct instruction *ip = t->ip;
6973         struct meter *m;
6974         uint64_t time, n_pkts, n_bytes;
6975         uint32_t length;
6976         enum rte_color color_in, color_out;
6977
6978         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
6979
6980         /* Structs. */
6981         m = instr_meter_idx_imm(p, ip);
6982         rte_prefetch0(m->n_pkts);
6983         time = rte_get_tsc_cycles();
6984         length = instr_meter_length_hbo(t, ip);
6985         color_in = instr_meter_color_in_hbo(t, ip);
6986
6987         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6988                 &m->profile->profile,
6989                 time,
6990                 length,
6991                 color_in);
6992
6993         color_out &= m->color_mask;
6994
6995         n_pkts = m->n_pkts[color_out];
6996         n_bytes = m->n_bytes[color_out];
6997
6998         instr_meter_color_out_hbo_set(t, ip, color_out);
6999
7000         m->n_pkts[color_out] = n_pkts + 1;
7001         m->n_bytes[color_out] = n_bytes + length;
7002
7003         /* Thread. */
7004         thread_ip_inc(p);
7005 }
7006 static inline void
7007 instr_meter_imi_exec(struct rte_swx_pipeline *p)
7008 {
7009         struct thread *t = &p->threads[p->thread_id];
7010         struct instruction *ip = t->ip;
7011         struct meter *m;
7012         uint64_t time, n_pkts, n_bytes;
7013         uint32_t length;
7014         enum rte_color color_in, color_out;
7015
7016         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
7017
7018         /* Structs. */
7019         m = instr_meter_idx_imm(p, ip);
7020         rte_prefetch0(m->n_pkts);
7021         time = rte_get_tsc_cycles();
7022         length = instr_meter_length_hbo(t, ip);
7023         color_in = (enum rte_color)ip->meter.color_in_val;
7024
7025         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7026                 &m->profile->profile,
7027                 time,
7028                 length,
7029                 color_in);
7030
7031         color_out &= m->color_mask;
7032
7033         n_pkts = m->n_pkts[color_out];
7034         n_bytes = m->n_bytes[color_out];
7035
7036         instr_meter_color_out_hbo_set(t, ip, color_out);
7037
7038         m->n_pkts[color_out] = n_pkts + 1;
7039         m->n_bytes[color_out] = n_bytes + length;
7040
7041         /* Thread. */
7042         thread_ip_inc(p);
7043 }
7044
7045 /*
7046  * jmp.
7047  */
7048 static struct action *
7049 action_find(struct rte_swx_pipeline *p, const char *name);
7050
7051 static int
7052 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
7053                     struct action *action __rte_unused,
7054                     char **tokens,
7055                     int n_tokens,
7056                     struct instruction *instr,
7057                     struct instruction_data *data)
7058 {
7059         CHECK(n_tokens == 2, EINVAL);
7060
7061         strcpy(data->jmp_label, tokens[1]);
7062
7063         instr->type = INSTR_JMP;
7064         instr->jmp.ip = NULL; /* Resolved later. */
7065         return 0;
7066 }
7067
7068 static int
7069 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
7070                           struct action *action __rte_unused,
7071                           char **tokens,
7072                           int n_tokens,
7073                           struct instruction *instr,
7074                           struct instruction_data *data)
7075 {
7076         struct header *h;
7077
7078         CHECK(n_tokens == 3, EINVAL);
7079
7080         strcpy(data->jmp_label, tokens[1]);
7081
7082         h = header_parse(p, tokens[2]);
7083         CHECK(h, EINVAL);
7084
7085         instr->type = INSTR_JMP_VALID;
7086         instr->jmp.ip = NULL; /* Resolved later. */
7087         instr->jmp.header_id = h->id;
7088         return 0;
7089 }
7090
7091 static int
7092 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
7093                             struct action *action __rte_unused,
7094                             char **tokens,
7095                             int n_tokens,
7096                             struct instruction *instr,
7097                             struct instruction_data *data)
7098 {
7099         struct header *h;
7100
7101         CHECK(n_tokens == 3, EINVAL);
7102
7103         strcpy(data->jmp_label, tokens[1]);
7104
7105         h = header_parse(p, tokens[2]);
7106         CHECK(h, EINVAL);
7107
7108         instr->type = INSTR_JMP_INVALID;
7109         instr->jmp.ip = NULL; /* Resolved later. */
7110         instr->jmp.header_id = h->id;
7111         return 0;
7112 }
7113
7114 static int
7115 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
7116                         struct action *action,
7117                         char **tokens,
7118                         int n_tokens,
7119                         struct instruction *instr,
7120                         struct instruction_data *data)
7121 {
7122         CHECK(!action, EINVAL);
7123         CHECK(n_tokens == 2, EINVAL);
7124
7125         strcpy(data->jmp_label, tokens[1]);
7126
7127         instr->type = INSTR_JMP_HIT;
7128         instr->jmp.ip = NULL; /* Resolved later. */
7129         return 0;
7130 }
7131
7132 static int
7133 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
7134                          struct action *action,
7135                          char **tokens,
7136                          int n_tokens,
7137                          struct instruction *instr,
7138                          struct instruction_data *data)
7139 {
7140         CHECK(!action, EINVAL);
7141         CHECK(n_tokens == 2, EINVAL);
7142
7143         strcpy(data->jmp_label, tokens[1]);
7144
7145         instr->type = INSTR_JMP_MISS;
7146         instr->jmp.ip = NULL; /* Resolved later. */
7147         return 0;
7148 }
7149
7150 static int
7151 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
7152                                struct action *action,
7153                                char **tokens,
7154                                int n_tokens,
7155                                struct instruction *instr,
7156                                struct instruction_data *data)
7157 {
7158         struct action *a;
7159
7160         CHECK(!action, EINVAL);
7161         CHECK(n_tokens == 3, EINVAL);
7162
7163         strcpy(data->jmp_label, tokens[1]);
7164
7165         a = action_find(p, tokens[2]);
7166         CHECK(a, EINVAL);
7167
7168         instr->type = INSTR_JMP_ACTION_HIT;
7169         instr->jmp.ip = NULL; /* Resolved later. */
7170         instr->jmp.action_id = a->id;
7171         return 0;
7172 }
7173
7174 static int
7175 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
7176                                 struct action *action,
7177                                 char **tokens,
7178                                 int n_tokens,
7179                                 struct instruction *instr,
7180                                 struct instruction_data *data)
7181 {
7182         struct action *a;
7183
7184         CHECK(!action, EINVAL);
7185         CHECK(n_tokens == 3, EINVAL);
7186
7187         strcpy(data->jmp_label, tokens[1]);
7188
7189         a = action_find(p, tokens[2]);
7190         CHECK(a, EINVAL);
7191
7192         instr->type = INSTR_JMP_ACTION_MISS;
7193         instr->jmp.ip = NULL; /* Resolved later. */
7194         instr->jmp.action_id = a->id;
7195         return 0;
7196 }
7197
7198 static int
7199 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
7200                        struct action *action,
7201                        char **tokens,
7202                        int n_tokens,
7203                        struct instruction *instr,
7204                        struct instruction_data *data)
7205 {
7206         char *a = tokens[2], *b = tokens[3];
7207         struct field *fa, *fb;
7208         uint64_t b_val;
7209         uint32_t a_struct_id, b_struct_id;
7210
7211         CHECK(n_tokens == 4, EINVAL);
7212
7213         strcpy(data->jmp_label, tokens[1]);
7214
7215         fa = struct_field_parse(p, action, a, &a_struct_id);
7216         CHECK(fa, EINVAL);
7217         CHECK(!fa->var_size, EINVAL);
7218
7219         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
7220         fb = struct_field_parse(p, action, b, &b_struct_id);
7221         if (fb) {
7222                 CHECK(!fb->var_size, EINVAL);
7223
7224                 instr->type = INSTR_JMP_EQ;
7225                 if (a[0] != 'h' && b[0] == 'h')
7226                         instr->type = INSTR_JMP_EQ_MH;
7227                 if (a[0] == 'h' && b[0] != 'h')
7228                         instr->type = INSTR_JMP_EQ_HM;
7229                 if (a[0] == 'h' && b[0] == 'h')
7230                         instr->type = INSTR_JMP_EQ_HH;
7231                 instr->jmp.ip = NULL; /* Resolved later. */
7232
7233                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7234                 instr->jmp.a.n_bits = fa->n_bits;
7235                 instr->jmp.a.offset = fa->offset / 8;
7236                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7237                 instr->jmp.b.n_bits = fb->n_bits;
7238                 instr->jmp.b.offset = fb->offset / 8;
7239                 return 0;
7240         }
7241
7242         /* JMP_EQ_I. */
7243         b_val = strtoull(b, &b, 0);
7244         CHECK(!b[0], EINVAL);
7245
7246         if (a[0] == 'h')
7247                 b_val = hton64(b_val) >> (64 - fa->n_bits);
7248
7249         instr->type = INSTR_JMP_EQ_I;
7250         instr->jmp.ip = NULL; /* Resolved later. */
7251         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7252         instr->jmp.a.n_bits = fa->n_bits;
7253         instr->jmp.a.offset = fa->offset / 8;
7254         instr->jmp.b_val = b_val;
7255         return 0;
7256 }
7257
7258 static int
7259 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
7260                         struct action *action,
7261                         char **tokens,
7262                         int n_tokens,
7263                         struct instruction *instr,
7264                         struct instruction_data *data)
7265 {
7266         char *a = tokens[2], *b = tokens[3];
7267         struct field *fa, *fb;
7268         uint64_t b_val;
7269         uint32_t a_struct_id, b_struct_id;
7270
7271         CHECK(n_tokens == 4, EINVAL);
7272
7273         strcpy(data->jmp_label, tokens[1]);
7274
7275         fa = struct_field_parse(p, action, a, &a_struct_id);
7276         CHECK(fa, EINVAL);
7277         CHECK(!fa->var_size, EINVAL);
7278
7279         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
7280         fb = struct_field_parse(p, action, b, &b_struct_id);
7281         if (fb) {
7282                 CHECK(!fb->var_size, EINVAL);
7283
7284                 instr->type = INSTR_JMP_NEQ;
7285                 if (a[0] != 'h' && b[0] == 'h')
7286                         instr->type = INSTR_JMP_NEQ_MH;
7287                 if (a[0] == 'h' && b[0] != 'h')
7288                         instr->type = INSTR_JMP_NEQ_HM;
7289                 if (a[0] == 'h' && b[0] == 'h')
7290                         instr->type = INSTR_JMP_NEQ_HH;
7291                 instr->jmp.ip = NULL; /* Resolved later. */
7292
7293                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7294                 instr->jmp.a.n_bits = fa->n_bits;
7295                 instr->jmp.a.offset = fa->offset / 8;
7296                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7297                 instr->jmp.b.n_bits = fb->n_bits;
7298                 instr->jmp.b.offset = fb->offset / 8;
7299                 return 0;
7300         }
7301
7302         /* JMP_NEQ_I. */
7303         b_val = strtoull(b, &b, 0);
7304         CHECK(!b[0], EINVAL);
7305
7306         if (a[0] == 'h')
7307                 b_val = hton64(b_val) >> (64 - fa->n_bits);
7308
7309         instr->type = INSTR_JMP_NEQ_I;
7310         instr->jmp.ip = NULL; /* Resolved later. */
7311         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7312         instr->jmp.a.n_bits = fa->n_bits;
7313         instr->jmp.a.offset = fa->offset / 8;
7314         instr->jmp.b_val = b_val;
7315         return 0;
7316 }
7317
7318 static int
7319 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
7320                        struct action *action,
7321                        char **tokens,
7322                        int n_tokens,
7323                        struct instruction *instr,
7324                        struct instruction_data *data)
7325 {
7326         char *a = tokens[2], *b = tokens[3];
7327         struct field *fa, *fb;
7328         uint64_t b_val;
7329         uint32_t a_struct_id, b_struct_id;
7330
7331         CHECK(n_tokens == 4, EINVAL);
7332
7333         strcpy(data->jmp_label, tokens[1]);
7334
7335         fa = struct_field_parse(p, action, a, &a_struct_id);
7336         CHECK(fa, EINVAL);
7337         CHECK(!fa->var_size, EINVAL);
7338
7339         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
7340         fb = struct_field_parse(p, action, b, &b_struct_id);
7341         if (fb) {
7342                 CHECK(!fb->var_size, EINVAL);
7343
7344                 instr->type = INSTR_JMP_LT;
7345                 if (a[0] == 'h' && b[0] != 'h')
7346                         instr->type = INSTR_JMP_LT_HM;
7347                 if (a[0] != 'h' && b[0] == 'h')
7348                         instr->type = INSTR_JMP_LT_MH;
7349                 if (a[0] == 'h' && b[0] == 'h')
7350                         instr->type = INSTR_JMP_LT_HH;
7351                 instr->jmp.ip = NULL; /* Resolved later. */
7352
7353                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7354                 instr->jmp.a.n_bits = fa->n_bits;
7355                 instr->jmp.a.offset = fa->offset / 8;
7356                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7357                 instr->jmp.b.n_bits = fb->n_bits;
7358                 instr->jmp.b.offset = fb->offset / 8;
7359                 return 0;
7360         }
7361
7362         /* JMP_LT_MI, JMP_LT_HI. */
7363         b_val = strtoull(b, &b, 0);
7364         CHECK(!b[0], EINVAL);
7365
7366         instr->type = INSTR_JMP_LT_MI;
7367         if (a[0] == 'h')
7368                 instr->type = INSTR_JMP_LT_HI;
7369         instr->jmp.ip = NULL; /* Resolved later. */
7370
7371         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7372         instr->jmp.a.n_bits = fa->n_bits;
7373         instr->jmp.a.offset = fa->offset / 8;
7374         instr->jmp.b_val = b_val;
7375         return 0;
7376 }
7377
7378 static int
7379 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
7380                        struct action *action,
7381                        char **tokens,
7382                        int n_tokens,
7383                        struct instruction *instr,
7384                        struct instruction_data *data)
7385 {
7386         char *a = tokens[2], *b = tokens[3];
7387         struct field *fa, *fb;
7388         uint64_t b_val;
7389         uint32_t a_struct_id, b_struct_id;
7390
7391         CHECK(n_tokens == 4, EINVAL);
7392
7393         strcpy(data->jmp_label, tokens[1]);
7394
7395         fa = struct_field_parse(p, action, a, &a_struct_id);
7396         CHECK(fa, EINVAL);
7397         CHECK(!fa->var_size, EINVAL);
7398
7399         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
7400         fb = struct_field_parse(p, action, b, &b_struct_id);
7401         if (fb) {
7402                 CHECK(!fb->var_size, EINVAL);
7403
7404                 instr->type = INSTR_JMP_GT;
7405                 if (a[0] == 'h' && b[0] != 'h')
7406                         instr->type = INSTR_JMP_GT_HM;
7407                 if (a[0] != 'h' && b[0] == 'h')
7408                         instr->type = INSTR_JMP_GT_MH;
7409                 if (a[0] == 'h' && b[0] == 'h')
7410                         instr->type = INSTR_JMP_GT_HH;
7411                 instr->jmp.ip = NULL; /* Resolved later. */
7412
7413                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7414                 instr->jmp.a.n_bits = fa->n_bits;
7415                 instr->jmp.a.offset = fa->offset / 8;
7416                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7417                 instr->jmp.b.n_bits = fb->n_bits;
7418                 instr->jmp.b.offset = fb->offset / 8;
7419                 return 0;
7420         }
7421
7422         /* JMP_GT_MI, JMP_GT_HI. */
7423         b_val = strtoull(b, &b, 0);
7424         CHECK(!b[0], EINVAL);
7425
7426         instr->type = INSTR_JMP_GT_MI;
7427         if (a[0] == 'h')
7428                 instr->type = INSTR_JMP_GT_HI;
7429         instr->jmp.ip = NULL; /* Resolved later. */
7430
7431         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7432         instr->jmp.a.n_bits = fa->n_bits;
7433         instr->jmp.a.offset = fa->offset / 8;
7434         instr->jmp.b_val = b_val;
7435         return 0;
7436 }
7437
7438 static inline void
7439 instr_jmp_exec(struct rte_swx_pipeline *p)
7440 {
7441         struct thread *t = &p->threads[p->thread_id];
7442         struct instruction *ip = t->ip;
7443
7444         TRACE("[Thread %2u] jmp\n", p->thread_id);
7445
7446         thread_ip_set(t, ip->jmp.ip);
7447 }
7448
7449 static inline void
7450 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
7451 {
7452         struct thread *t = &p->threads[p->thread_id];
7453         struct instruction *ip = t->ip;
7454         uint32_t header_id = ip->jmp.header_id;
7455
7456         TRACE("[Thread %2u] jmpv\n", p->thread_id);
7457
7458         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
7459 }
7460
7461 static inline void
7462 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
7463 {
7464         struct thread *t = &p->threads[p->thread_id];
7465         struct instruction *ip = t->ip;
7466         uint32_t header_id = ip->jmp.header_id;
7467
7468         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
7469
7470         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
7471 }
7472
7473 static inline void
7474 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
7475 {
7476         struct thread *t = &p->threads[p->thread_id];
7477         struct instruction *ip = t->ip;
7478         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
7479
7480         TRACE("[Thread %2u] jmph\n", p->thread_id);
7481
7482         t->ip = ip_next[t->hit];
7483 }
7484
7485 static inline void
7486 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
7487 {
7488         struct thread *t = &p->threads[p->thread_id];
7489         struct instruction *ip = t->ip;
7490         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
7491
7492         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
7493
7494         t->ip = ip_next[t->hit];
7495 }
7496
7497 static inline void
7498 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
7499 {
7500         struct thread *t = &p->threads[p->thread_id];
7501         struct instruction *ip = t->ip;
7502
7503         TRACE("[Thread %2u] jmpa\n", p->thread_id);
7504
7505         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
7506 }
7507
7508 static inline void
7509 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
7510 {
7511         struct thread *t = &p->threads[p->thread_id];
7512         struct instruction *ip = t->ip;
7513
7514         TRACE("[Thread %2u] jmpna\n", p->thread_id);
7515
7516         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
7517 }
7518
7519 static inline void
7520 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
7521 {
7522         struct thread *t = &p->threads[p->thread_id];
7523         struct instruction *ip = t->ip;
7524
7525         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
7526
7527         JMP_CMP(t, ip, ==);
7528 }
7529
7530 static inline void
7531 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
7532 {
7533         struct thread *t = &p->threads[p->thread_id];
7534         struct instruction *ip = t->ip;
7535
7536         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
7537
7538         JMP_CMP_MH(t, ip, ==);
7539 }
7540
7541 static inline void
7542 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
7543 {
7544         struct thread *t = &p->threads[p->thread_id];
7545         struct instruction *ip = t->ip;
7546
7547         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
7548
7549         JMP_CMP_HM(t, ip, ==);
7550 }
7551
7552 static inline void
7553 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
7554 {
7555         struct thread *t = &p->threads[p->thread_id];
7556         struct instruction *ip = t->ip;
7557
7558         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
7559
7560         JMP_CMP_HH_FAST(t, ip, ==);
7561 }
7562
7563 static inline void
7564 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
7565 {
7566         struct thread *t = &p->threads[p->thread_id];
7567         struct instruction *ip = t->ip;
7568
7569         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
7570
7571         JMP_CMP_I(t, ip, ==);
7572 }
7573
7574 static inline void
7575 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
7576 {
7577         struct thread *t = &p->threads[p->thread_id];
7578         struct instruction *ip = t->ip;
7579
7580         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
7581
7582         JMP_CMP(t, ip, !=);
7583 }
7584
7585 static inline void
7586 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
7587 {
7588         struct thread *t = &p->threads[p->thread_id];
7589         struct instruction *ip = t->ip;
7590
7591         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
7592
7593         JMP_CMP_MH(t, ip, !=);
7594 }
7595
7596 static inline void
7597 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
7598 {
7599         struct thread *t = &p->threads[p->thread_id];
7600         struct instruction *ip = t->ip;
7601
7602         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
7603
7604         JMP_CMP_HM(t, ip, !=);
7605 }
7606
7607 static inline void
7608 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
7609 {
7610         struct thread *t = &p->threads[p->thread_id];
7611         struct instruction *ip = t->ip;
7612
7613         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
7614
7615         JMP_CMP_HH_FAST(t, ip, !=);
7616 }
7617
7618 static inline void
7619 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
7620 {
7621         struct thread *t = &p->threads[p->thread_id];
7622         struct instruction *ip = t->ip;
7623
7624         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
7625
7626         JMP_CMP_I(t, ip, !=);
7627 }
7628
7629 static inline void
7630 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
7631 {
7632         struct thread *t = &p->threads[p->thread_id];
7633         struct instruction *ip = t->ip;
7634
7635         TRACE("[Thread %2u] jmplt\n", p->thread_id);
7636
7637         JMP_CMP(t, ip, <);
7638 }
7639
7640 static inline void
7641 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
7642 {
7643         struct thread *t = &p->threads[p->thread_id];
7644         struct instruction *ip = t->ip;
7645
7646         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
7647
7648         JMP_CMP_MH(t, ip, <);
7649 }
7650
7651 static inline void
7652 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
7653 {
7654         struct thread *t = &p->threads[p->thread_id];
7655         struct instruction *ip = t->ip;
7656
7657         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
7658
7659         JMP_CMP_HM(t, ip, <);
7660 }
7661
7662 static inline void
7663 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
7664 {
7665         struct thread *t = &p->threads[p->thread_id];
7666         struct instruction *ip = t->ip;
7667
7668         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
7669
7670         JMP_CMP_HH(t, ip, <);
7671 }
7672
7673 static inline void
7674 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
7675 {
7676         struct thread *t = &p->threads[p->thread_id];
7677         struct instruction *ip = t->ip;
7678
7679         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
7680
7681         JMP_CMP_MI(t, ip, <);
7682 }
7683
7684 static inline void
7685 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
7686 {
7687         struct thread *t = &p->threads[p->thread_id];
7688         struct instruction *ip = t->ip;
7689
7690         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
7691
7692         JMP_CMP_HI(t, ip, <);
7693 }
7694
7695 static inline void
7696 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
7697 {
7698         struct thread *t = &p->threads[p->thread_id];
7699         struct instruction *ip = t->ip;
7700
7701         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
7702
7703         JMP_CMP(t, ip, >);
7704 }
7705
7706 static inline void
7707 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
7708 {
7709         struct thread *t = &p->threads[p->thread_id];
7710         struct instruction *ip = t->ip;
7711
7712         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
7713
7714         JMP_CMP_MH(t, ip, >);
7715 }
7716
7717 static inline void
7718 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
7719 {
7720         struct thread *t = &p->threads[p->thread_id];
7721         struct instruction *ip = t->ip;
7722
7723         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
7724
7725         JMP_CMP_HM(t, ip, >);
7726 }
7727
7728 static inline void
7729 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
7730 {
7731         struct thread *t = &p->threads[p->thread_id];
7732         struct instruction *ip = t->ip;
7733
7734         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
7735
7736         JMP_CMP_HH(t, ip, >);
7737 }
7738
7739 static inline void
7740 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
7741 {
7742         struct thread *t = &p->threads[p->thread_id];
7743         struct instruction *ip = t->ip;
7744
7745         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
7746
7747         JMP_CMP_MI(t, ip, >);
7748 }
7749
7750 static inline void
7751 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
7752 {
7753         struct thread *t = &p->threads[p->thread_id];
7754         struct instruction *ip = t->ip;
7755
7756         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
7757
7758         JMP_CMP_HI(t, ip, >);
7759 }
7760
7761 /*
7762  * return.
7763  */
7764 static int
7765 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
7766                        struct action *action,
7767                        char **tokens __rte_unused,
7768                        int n_tokens,
7769                        struct instruction *instr,
7770                        struct instruction_data *data __rte_unused)
7771 {
7772         CHECK(action, EINVAL);
7773         CHECK(n_tokens == 1, EINVAL);
7774
7775         instr->type = INSTR_RETURN;
7776         return 0;
7777 }
7778
7779 static inline void
7780 instr_return_exec(struct rte_swx_pipeline *p)
7781 {
7782         struct thread *t = &p->threads[p->thread_id];
7783
7784         TRACE("[Thread %2u] return\n", p->thread_id);
7785
7786         t->ip = t->ret;
7787 }
7788
7789 static int
7790 instr_translate(struct rte_swx_pipeline *p,
7791                 struct action *action,
7792                 char *string,
7793                 struct instruction *instr,
7794                 struct instruction_data *data)
7795 {
7796         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
7797         int n_tokens = 0, tpos = 0;
7798
7799         /* Parse the instruction string into tokens. */
7800         for ( ; ; ) {
7801                 char *token;
7802
7803                 token = strtok_r(string, " \t\v", &string);
7804                 if (!token)
7805                         break;
7806
7807                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
7808                 CHECK_NAME(token, EINVAL);
7809
7810                 tokens[n_tokens] = token;
7811                 n_tokens++;
7812         }
7813
7814         CHECK(n_tokens, EINVAL);
7815
7816         /* Handle the optional instruction label. */
7817         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
7818                 strcpy(data->label, tokens[0]);
7819
7820                 tpos += 2;
7821                 CHECK(n_tokens - tpos, EINVAL);
7822         }
7823
7824         /* Identify the instruction type. */
7825         if (!strcmp(tokens[tpos], "rx"))
7826                 return instr_rx_translate(p,
7827                                           action,
7828                                           &tokens[tpos],
7829                                           n_tokens - tpos,
7830                                           instr,
7831                                           data);
7832
7833         if (!strcmp(tokens[tpos], "tx"))
7834                 return instr_tx_translate(p,
7835                                           action,
7836                                           &tokens[tpos],
7837                                           n_tokens - tpos,
7838                                           instr,
7839                                           data);
7840
7841         if (!strcmp(tokens[tpos], "drop"))
7842                 return instr_drop_translate(p,
7843                                             action,
7844                                             &tokens[tpos],
7845                                             n_tokens - tpos,
7846                                             instr,
7847                                             data);
7848
7849         if (!strcmp(tokens[tpos], "extract"))
7850                 return instr_hdr_extract_translate(p,
7851                                                    action,
7852                                                    &tokens[tpos],
7853                                                    n_tokens - tpos,
7854                                                    instr,
7855                                                    data);
7856
7857         if (!strcmp(tokens[tpos], "emit"))
7858                 return instr_hdr_emit_translate(p,
7859                                                 action,
7860                                                 &tokens[tpos],
7861                                                 n_tokens - tpos,
7862                                                 instr,
7863                                                 data);
7864
7865         if (!strcmp(tokens[tpos], "validate"))
7866                 return instr_hdr_validate_translate(p,
7867                                                     action,
7868                                                     &tokens[tpos],
7869                                                     n_tokens - tpos,
7870                                                     instr,
7871                                                     data);
7872
7873         if (!strcmp(tokens[tpos], "invalidate"))
7874                 return instr_hdr_invalidate_translate(p,
7875                                                       action,
7876                                                       &tokens[tpos],
7877                                                       n_tokens - tpos,
7878                                                       instr,
7879                                                       data);
7880
7881         if (!strcmp(tokens[tpos], "mov"))
7882                 return instr_mov_translate(p,
7883                                            action,
7884                                            &tokens[tpos],
7885                                            n_tokens - tpos,
7886                                            instr,
7887                                            data);
7888
7889         if (!strcmp(tokens[tpos], "add"))
7890                 return instr_alu_add_translate(p,
7891                                                action,
7892                                                &tokens[tpos],
7893                                                n_tokens - tpos,
7894                                                instr,
7895                                                data);
7896
7897         if (!strcmp(tokens[tpos], "sub"))
7898                 return instr_alu_sub_translate(p,
7899                                                action,
7900                                                &tokens[tpos],
7901                                                n_tokens - tpos,
7902                                                instr,
7903                                                data);
7904
7905         if (!strcmp(tokens[tpos], "ckadd"))
7906                 return instr_alu_ckadd_translate(p,
7907                                                  action,
7908                                                  &tokens[tpos],
7909                                                  n_tokens - tpos,
7910                                                  instr,
7911                                                  data);
7912
7913         if (!strcmp(tokens[tpos], "cksub"))
7914                 return instr_alu_cksub_translate(p,
7915                                                  action,
7916                                                  &tokens[tpos],
7917                                                  n_tokens - tpos,
7918                                                  instr,
7919                                                  data);
7920
7921         if (!strcmp(tokens[tpos], "and"))
7922                 return instr_alu_and_translate(p,
7923                                                action,
7924                                                &tokens[tpos],
7925                                                n_tokens - tpos,
7926                                                instr,
7927                                                data);
7928
7929         if (!strcmp(tokens[tpos], "or"))
7930                 return instr_alu_or_translate(p,
7931                                               action,
7932                                               &tokens[tpos],
7933                                               n_tokens - tpos,
7934                                               instr,
7935                                               data);
7936
7937         if (!strcmp(tokens[tpos], "xor"))
7938                 return instr_alu_xor_translate(p,
7939                                                action,
7940                                                &tokens[tpos],
7941                                                n_tokens - tpos,
7942                                                instr,
7943                                                data);
7944
7945         if (!strcmp(tokens[tpos], "shl"))
7946                 return instr_alu_shl_translate(p,
7947                                                action,
7948                                                &tokens[tpos],
7949                                                n_tokens - tpos,
7950                                                instr,
7951                                                data);
7952
7953         if (!strcmp(tokens[tpos], "shr"))
7954                 return instr_alu_shr_translate(p,
7955                                                action,
7956                                                &tokens[tpos],
7957                                                n_tokens - tpos,
7958                                                instr,
7959                                                data);
7960
7961         if (!strcmp(tokens[tpos], "regprefetch"))
7962                 return instr_regprefetch_translate(p,
7963                                                    action,
7964                                                    &tokens[tpos],
7965                                                    n_tokens - tpos,
7966                                                    instr,
7967                                                    data);
7968
7969         if (!strcmp(tokens[tpos], "regrd"))
7970                 return instr_regrd_translate(p,
7971                                              action,
7972                                              &tokens[tpos],
7973                                              n_tokens - tpos,
7974                                              instr,
7975                                              data);
7976
7977         if (!strcmp(tokens[tpos], "regwr"))
7978                 return instr_regwr_translate(p,
7979                                              action,
7980                                              &tokens[tpos],
7981                                              n_tokens - tpos,
7982                                              instr,
7983                                              data);
7984
7985         if (!strcmp(tokens[tpos], "regadd"))
7986                 return instr_regadd_translate(p,
7987                                               action,
7988                                               &tokens[tpos],
7989                                               n_tokens - tpos,
7990                                               instr,
7991                                               data);
7992
7993         if (!strcmp(tokens[tpos], "metprefetch"))
7994                 return instr_metprefetch_translate(p,
7995                                                    action,
7996                                                    &tokens[tpos],
7997                                                    n_tokens - tpos,
7998                                                    instr,
7999                                                    data);
8000
8001         if (!strcmp(tokens[tpos], "meter"))
8002                 return instr_meter_translate(p,
8003                                              action,
8004                                              &tokens[tpos],
8005                                              n_tokens - tpos,
8006                                              instr,
8007                                              data);
8008
8009         if (!strcmp(tokens[tpos], "table"))
8010                 return instr_table_translate(p,
8011                                              action,
8012                                              &tokens[tpos],
8013                                              n_tokens - tpos,
8014                                              instr,
8015                                              data);
8016
8017         if (!strcmp(tokens[tpos], "extern"))
8018                 return instr_extern_translate(p,
8019                                               action,
8020                                               &tokens[tpos],
8021                                               n_tokens - tpos,
8022                                               instr,
8023                                               data);
8024
8025         if (!strcmp(tokens[tpos], "jmp"))
8026                 return instr_jmp_translate(p,
8027                                            action,
8028                                            &tokens[tpos],
8029                                            n_tokens - tpos,
8030                                            instr,
8031                                            data);
8032
8033         if (!strcmp(tokens[tpos], "jmpv"))
8034                 return instr_jmp_valid_translate(p,
8035                                                  action,
8036                                                  &tokens[tpos],
8037                                                  n_tokens - tpos,
8038                                                  instr,
8039                                                  data);
8040
8041         if (!strcmp(tokens[tpos], "jmpnv"))
8042                 return instr_jmp_invalid_translate(p,
8043                                                    action,
8044                                                    &tokens[tpos],
8045                                                    n_tokens - tpos,
8046                                                    instr,
8047                                                    data);
8048
8049         if (!strcmp(tokens[tpos], "jmph"))
8050                 return instr_jmp_hit_translate(p,
8051                                                action,
8052                                                &tokens[tpos],
8053                                                n_tokens - tpos,
8054                                                instr,
8055                                                data);
8056
8057         if (!strcmp(tokens[tpos], "jmpnh"))
8058                 return instr_jmp_miss_translate(p,
8059                                                 action,
8060                                                 &tokens[tpos],
8061                                                 n_tokens - tpos,
8062                                                 instr,
8063                                                 data);
8064
8065         if (!strcmp(tokens[tpos], "jmpa"))
8066                 return instr_jmp_action_hit_translate(p,
8067                                                       action,
8068                                                       &tokens[tpos],
8069                                                       n_tokens - tpos,
8070                                                       instr,
8071                                                       data);
8072
8073         if (!strcmp(tokens[tpos], "jmpna"))
8074                 return instr_jmp_action_miss_translate(p,
8075                                                        action,
8076                                                        &tokens[tpos],
8077                                                        n_tokens - tpos,
8078                                                        instr,
8079                                                        data);
8080
8081         if (!strcmp(tokens[tpos], "jmpeq"))
8082                 return instr_jmp_eq_translate(p,
8083                                               action,
8084                                               &tokens[tpos],
8085                                               n_tokens - tpos,
8086                                               instr,
8087                                               data);
8088
8089         if (!strcmp(tokens[tpos], "jmpneq"))
8090                 return instr_jmp_neq_translate(p,
8091                                                action,
8092                                                &tokens[tpos],
8093                                                n_tokens - tpos,
8094                                                instr,
8095                                                data);
8096
8097         if (!strcmp(tokens[tpos], "jmplt"))
8098                 return instr_jmp_lt_translate(p,
8099                                               action,
8100                                               &tokens[tpos],
8101                                               n_tokens - tpos,
8102                                               instr,
8103                                               data);
8104
8105         if (!strcmp(tokens[tpos], "jmpgt"))
8106                 return instr_jmp_gt_translate(p,
8107                                               action,
8108                                               &tokens[tpos],
8109                                               n_tokens - tpos,
8110                                               instr,
8111                                               data);
8112
8113         if (!strcmp(tokens[tpos], "return"))
8114                 return instr_return_translate(p,
8115                                               action,
8116                                               &tokens[tpos],
8117                                               n_tokens - tpos,
8118                                               instr,
8119                                               data);
8120
8121         CHECK(0, EINVAL);
8122 }
8123
8124 static struct instruction_data *
8125 label_find(struct instruction_data *data, uint32_t n, const char *label)
8126 {
8127         uint32_t i;
8128
8129         for (i = 0; i < n; i++)
8130                 if (!strcmp(label, data[i].label))
8131                         return &data[i];
8132
8133         return NULL;
8134 }
8135
8136 static uint32_t
8137 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
8138 {
8139         uint32_t count = 0, i;
8140
8141         if (!label[0])
8142                 return 0;
8143
8144         for (i = 0; i < n; i++)
8145                 if (!strcmp(label, data[i].jmp_label))
8146                         count++;
8147
8148         return count;
8149 }
8150
8151 static int
8152 instr_label_check(struct instruction_data *instruction_data,
8153                   uint32_t n_instructions)
8154 {
8155         uint32_t i;
8156
8157         /* Check that all instruction labels are unique. */
8158         for (i = 0; i < n_instructions; i++) {
8159                 struct instruction_data *data = &instruction_data[i];
8160                 char *label = data->label;
8161                 uint32_t j;
8162
8163                 if (!label[0])
8164                         continue;
8165
8166                 for (j = i + 1; j < n_instructions; j++)
8167                         CHECK(strcmp(label, data[j].label), EINVAL);
8168         }
8169
8170         /* Get users for each instruction label. */
8171         for (i = 0; i < n_instructions; i++) {
8172                 struct instruction_data *data = &instruction_data[i];
8173                 char *label = data->label;
8174
8175                 data->n_users = label_is_used(instruction_data,
8176                                               n_instructions,
8177                                               label);
8178         }
8179
8180         return 0;
8181 }
8182
8183 static int
8184 instr_jmp_resolve(struct instruction *instructions,
8185                   struct instruction_data *instruction_data,
8186                   uint32_t n_instructions)
8187 {
8188         uint32_t i;
8189
8190         for (i = 0; i < n_instructions; i++) {
8191                 struct instruction *instr = &instructions[i];
8192                 struct instruction_data *data = &instruction_data[i];
8193                 struct instruction_data *found;
8194
8195                 if (!instruction_is_jmp(instr))
8196                         continue;
8197
8198                 found = label_find(instruction_data,
8199                                    n_instructions,
8200                                    data->jmp_label);
8201                 CHECK(found, EINVAL);
8202
8203                 instr->jmp.ip = &instructions[found - instruction_data];
8204         }
8205
8206         return 0;
8207 }
8208
8209 static int
8210 instr_verify(struct rte_swx_pipeline *p __rte_unused,
8211              struct action *a,
8212              struct instruction *instr,
8213              struct instruction_data *data __rte_unused,
8214              uint32_t n_instructions)
8215 {
8216         if (!a) {
8217                 enum instruction_type type;
8218                 uint32_t i;
8219
8220                 /* Check that the first instruction is rx. */
8221                 CHECK(instr[0].type == INSTR_RX, EINVAL);
8222
8223                 /* Check that there is at least one tx instruction. */
8224                 for (i = 0; i < n_instructions; i++) {
8225                         type = instr[i].type;
8226
8227                         if (instruction_is_tx(type))
8228                                 break;
8229                 }
8230                 CHECK(i < n_instructions, EINVAL);
8231
8232                 /* Check that the last instruction is either tx or unconditional
8233                  * jump.
8234                  */
8235                 type = instr[n_instructions - 1].type;
8236                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
8237         }
8238
8239         if (a) {
8240                 enum instruction_type type;
8241                 uint32_t i;
8242
8243                 /* Check that there is at least one return or tx instruction. */
8244                 for (i = 0; i < n_instructions; i++) {
8245                         type = instr[i].type;
8246
8247                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
8248                                 break;
8249                 }
8250                 CHECK(i < n_instructions, EINVAL);
8251         }
8252
8253         return 0;
8254 }
8255
8256 static uint32_t
8257 instr_compact(struct instruction *instructions,
8258               struct instruction_data *instruction_data,
8259               uint32_t n_instructions)
8260 {
8261         uint32_t i, pos = 0;
8262
8263         /* Eliminate the invalid instructions that have been optimized out. */
8264         for (i = 0; i < n_instructions; i++) {
8265                 struct instruction *instr = &instructions[i];
8266                 struct instruction_data *data = &instruction_data[i];
8267
8268                 if (data->invalid)
8269                         continue;
8270
8271                 if (i != pos) {
8272                         memcpy(&instructions[pos], instr, sizeof(*instr));
8273                         memcpy(&instruction_data[pos], data, sizeof(*data));
8274                 }
8275
8276                 pos++;
8277         }
8278
8279         return pos;
8280 }
8281
8282 static int
8283 instr_pattern_extract_many_search(struct instruction *instr,
8284                                   struct instruction_data *data,
8285                                   uint32_t n_instr,
8286                                   uint32_t *n_pattern_instr)
8287 {
8288         uint32_t i;
8289
8290         for (i = 0; i < n_instr; i++) {
8291                 if (data[i].invalid)
8292                         break;
8293
8294                 if (instr[i].type != INSTR_HDR_EXTRACT)
8295                         break;
8296
8297                 if (i == RTE_DIM(instr->io.hdr.header_id))
8298                         break;
8299
8300                 if (i && data[i].n_users)
8301                         break;
8302         }
8303
8304         if (i < 2)
8305                 return 0;
8306
8307         *n_pattern_instr = i;
8308         return 1;
8309 }
8310
8311 static void
8312 instr_pattern_extract_many_replace(struct instruction *instr,
8313                                    struct instruction_data *data,
8314                                    uint32_t n_instr)
8315 {
8316         uint32_t i;
8317
8318         for (i = 1; i < n_instr; i++) {
8319                 instr[0].type++;
8320                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8321                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8322                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8323
8324                 data[i].invalid = 1;
8325         }
8326 }
8327
8328 static uint32_t
8329 instr_pattern_extract_many_optimize(struct instruction *instructions,
8330                                     struct instruction_data *instruction_data,
8331                                     uint32_t n_instructions)
8332 {
8333         uint32_t i;
8334
8335         for (i = 0; i < n_instructions; ) {
8336                 struct instruction *instr = &instructions[i];
8337                 struct instruction_data *data = &instruction_data[i];
8338                 uint32_t n_instr = 0;
8339                 int detected;
8340
8341                 /* Extract many. */
8342                 detected = instr_pattern_extract_many_search(instr,
8343                                                              data,
8344                                                              n_instructions - i,
8345                                                              &n_instr);
8346                 if (detected) {
8347                         instr_pattern_extract_many_replace(instr,
8348                                                            data,
8349                                                            n_instr);
8350                         i += n_instr;
8351                         continue;
8352                 }
8353
8354                 /* No pattern starting at the current instruction. */
8355                 i++;
8356         }
8357
8358         /* Eliminate the invalid instructions that have been optimized out. */
8359         n_instructions = instr_compact(instructions,
8360                                        instruction_data,
8361                                        n_instructions);
8362
8363         return n_instructions;
8364 }
8365
8366 static int
8367 instr_pattern_emit_many_tx_search(struct instruction *instr,
8368                                   struct instruction_data *data,
8369                                   uint32_t n_instr,
8370                                   uint32_t *n_pattern_instr)
8371 {
8372         uint32_t i;
8373
8374         for (i = 0; i < n_instr; i++) {
8375                 if (data[i].invalid)
8376                         break;
8377
8378                 if (instr[i].type != INSTR_HDR_EMIT)
8379                         break;
8380
8381                 if (i == RTE_DIM(instr->io.hdr.header_id))
8382                         break;
8383
8384                 if (i && data[i].n_users)
8385                         break;
8386         }
8387
8388         if (!i)
8389                 return 0;
8390
8391         if (!instruction_is_tx(instr[i].type))
8392                 return 0;
8393
8394         if (data[i].n_users)
8395                 return 0;
8396
8397         i++;
8398
8399         *n_pattern_instr = i;
8400         return 1;
8401 }
8402
8403 static void
8404 instr_pattern_emit_many_tx_replace(struct instruction *instr,
8405                                    struct instruction_data *data,
8406                                    uint32_t n_instr)
8407 {
8408         uint32_t i;
8409
8410         /* Any emit instruction in addition to the first one. */
8411         for (i = 1; i < n_instr - 1; i++) {
8412                 instr[0].type++;
8413                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8414                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8415                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8416
8417                 data[i].invalid = 1;
8418         }
8419
8420         /* The TX instruction is the last one in the pattern. */
8421         instr[0].type++;
8422         instr[0].io.io.offset = instr[i].io.io.offset;
8423         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
8424         data[i].invalid = 1;
8425 }
8426
8427 static uint32_t
8428 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
8429                                     struct instruction_data *instruction_data,
8430                                     uint32_t n_instructions)
8431 {
8432         uint32_t i;
8433
8434         for (i = 0; i < n_instructions; ) {
8435                 struct instruction *instr = &instructions[i];
8436                 struct instruction_data *data = &instruction_data[i];
8437                 uint32_t n_instr = 0;
8438                 int detected;
8439
8440                 /* Emit many + TX. */
8441                 detected = instr_pattern_emit_many_tx_search(instr,
8442                                                              data,
8443                                                              n_instructions - i,
8444                                                              &n_instr);
8445                 if (detected) {
8446                         instr_pattern_emit_many_tx_replace(instr,
8447                                                            data,
8448                                                            n_instr);
8449                         i += n_instr;
8450                         continue;
8451                 }
8452
8453                 /* No pattern starting at the current instruction. */
8454                 i++;
8455         }
8456
8457         /* Eliminate the invalid instructions that have been optimized out. */
8458         n_instructions = instr_compact(instructions,
8459                                        instruction_data,
8460                                        n_instructions);
8461
8462         return n_instructions;
8463 }
8464
8465 static uint32_t
8466 action_arg_src_mov_count(struct action *a,
8467                          uint32_t arg_id,
8468                          struct instruction *instructions,
8469                          struct instruction_data *instruction_data,
8470                          uint32_t n_instructions);
8471
8472 static int
8473 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
8474                                       struct action *a,
8475                                       struct instruction *instr,
8476                                       struct instruction_data *data,
8477                                       uint32_t n_instr,
8478                                       struct instruction *instructions,
8479                                       struct instruction_data *instruction_data,
8480                                       uint32_t n_instructions,
8481                                       uint32_t *n_pattern_instr)
8482 {
8483         struct header *h;
8484         uint32_t src_field_id, i, j;
8485
8486         /* Prerequisites. */
8487         if (!a || !a->st)
8488                 return 0;
8489
8490         /* First instruction: MOV_HM. */
8491         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
8492                 return 0;
8493
8494         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
8495         if (!h || h->st->var_size)
8496                 return 0;
8497
8498         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
8499                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
8500                         break;
8501
8502         if (src_field_id == a->st->n_fields)
8503                 return 0;
8504
8505         if (instr[0].mov.dst.offset ||
8506             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
8507             instr[0].mov.src.struct_id ||
8508             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
8509             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
8510                 return 0;
8511
8512         if ((n_instr < h->st->n_fields + 1) ||
8513              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
8514                 return 0;
8515
8516         /* Subsequent instructions: MOV_HM. */
8517         for (i = 1; i < h->st->n_fields; i++)
8518                 if (data[i].invalid ||
8519                     data[i].n_users ||
8520                     (instr[i].type != INSTR_MOV_HM) ||
8521                     (instr[i].mov.dst.struct_id != h->struct_id) ||
8522                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
8523                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
8524                     instr[i].mov.src.struct_id ||
8525                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
8526                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
8527                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
8528                         return 0;
8529
8530         /* Last instruction: HDR_VALIDATE. */
8531         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
8532             (instr[i].valid.header_id != h->id))
8533                 return 0;
8534
8535         /* Check that none of the action args that are used as source for this
8536          * DMA transfer are not used as source in any other mov instruction.
8537          */
8538         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
8539                 uint32_t n_users;
8540
8541                 n_users = action_arg_src_mov_count(a,
8542                                                    j,
8543                                                    instructions,
8544                                                    instruction_data,
8545                                                    n_instructions);
8546                 if (n_users > 1)
8547                         return 0;
8548         }
8549
8550         *n_pattern_instr = 1 + i;
8551         return 1;
8552 }
8553
8554 static void
8555 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
8556                                        struct action *a,
8557                                        struct instruction *instr,
8558                                        struct instruction_data *data,
8559                                        uint32_t n_instr)
8560 {
8561         struct header *h;
8562         uint32_t src_field_id, src_offset, i;
8563
8564         /* Read from the instructions before they are modified. */
8565         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
8566         if (!h)
8567                 return;
8568
8569         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
8570                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
8571                         break;
8572
8573         if (src_field_id == a->st->n_fields)
8574                 return;
8575
8576         src_offset = instr[0].mov.src.offset;
8577
8578         /* Modify the instructions. */
8579         instr[0].type = INSTR_DMA_HT;
8580         instr[0].dma.dst.header_id[0] = h->id;
8581         instr[0].dma.dst.struct_id[0] = h->struct_id;
8582         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
8583         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
8584
8585         for (i = 1; i < n_instr; i++)
8586                 data[i].invalid = 1;
8587
8588         /* Update the endianness of the action arguments to header endianness. */
8589         for (i = 0; i < h->st->n_fields; i++)
8590                 a->args_endianness[src_field_id + i] = 1;
8591 }
8592
8593 static uint32_t
8594 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
8595                                         struct action *a,
8596                                         struct instruction *instructions,
8597                                         struct instruction_data *instruction_data,
8598                                         uint32_t n_instructions)
8599 {
8600         uint32_t i;
8601
8602         if (!a || !a->st)
8603                 return n_instructions;
8604
8605         for (i = 0; i < n_instructions; ) {
8606                 struct instruction *instr = &instructions[i];
8607                 struct instruction_data *data = &instruction_data[i];
8608                 uint32_t n_instr = 0;
8609                 int detected;
8610
8611                 /* Mov all + validate. */
8612                 detected = instr_pattern_mov_all_validate_search(p,
8613                                                                  a,
8614                                                                  instr,
8615                                                                  data,
8616                                                                  n_instructions - i,
8617                                                                  instructions,
8618                                                                  instruction_data,
8619                                                                  n_instructions,
8620                                                                  &n_instr);
8621                 if (detected) {
8622                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
8623                         i += n_instr;
8624                         continue;
8625                 }
8626
8627                 /* No pattern starting at the current instruction. */
8628                 i++;
8629         }
8630
8631         /* Eliminate the invalid instructions that have been optimized out. */
8632         n_instructions = instr_compact(instructions,
8633                                        instruction_data,
8634                                        n_instructions);
8635
8636         return n_instructions;
8637 }
8638
8639 static int
8640 instr_pattern_dma_many_search(struct instruction *instr,
8641                               struct instruction_data *data,
8642                               uint32_t n_instr,
8643                               uint32_t *n_pattern_instr)
8644 {
8645         uint32_t i;
8646
8647         for (i = 0; i < n_instr; i++) {
8648                 if (data[i].invalid)
8649                         break;
8650
8651                 if (instr[i].type != INSTR_DMA_HT)
8652                         break;
8653
8654                 if (i == RTE_DIM(instr->dma.dst.header_id))
8655                         break;
8656
8657                 if (i && data[i].n_users)
8658                         break;
8659         }
8660
8661         if (i < 2)
8662                 return 0;
8663
8664         *n_pattern_instr = i;
8665         return 1;
8666 }
8667
8668 static void
8669 instr_pattern_dma_many_replace(struct instruction *instr,
8670                                struct instruction_data *data,
8671                                uint32_t n_instr)
8672 {
8673         uint32_t i;
8674
8675         for (i = 1; i < n_instr; i++) {
8676                 instr[0].type++;
8677                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
8678                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
8679                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
8680                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
8681
8682                 data[i].invalid = 1;
8683         }
8684 }
8685
8686 static uint32_t
8687 instr_pattern_dma_many_optimize(struct instruction *instructions,
8688                struct instruction_data *instruction_data,
8689                uint32_t n_instructions)
8690 {
8691         uint32_t i;
8692
8693         for (i = 0; i < n_instructions; ) {
8694                 struct instruction *instr = &instructions[i];
8695                 struct instruction_data *data = &instruction_data[i];
8696                 uint32_t n_instr = 0;
8697                 int detected;
8698
8699                 /* DMA many. */
8700                 detected = instr_pattern_dma_many_search(instr,
8701                                                          data,
8702                                                          n_instructions - i,
8703                                                          &n_instr);
8704                 if (detected) {
8705                         instr_pattern_dma_many_replace(instr, data, n_instr);
8706                         i += n_instr;
8707                         continue;
8708                 }
8709
8710                 /* No pattern starting at the current instruction. */
8711                 i++;
8712         }
8713
8714         /* Eliminate the invalid instructions that have been optimized out. */
8715         n_instructions = instr_compact(instructions,
8716                                        instruction_data,
8717                                        n_instructions);
8718
8719         return n_instructions;
8720 }
8721
8722 static uint32_t
8723 instr_optimize(struct rte_swx_pipeline *p,
8724                struct action *a,
8725                struct instruction *instructions,
8726                struct instruction_data *instruction_data,
8727                uint32_t n_instructions)
8728 {
8729         /* Extract many. */
8730         n_instructions = instr_pattern_extract_many_optimize(instructions,
8731                                                              instruction_data,
8732                                                              n_instructions);
8733
8734         /* Emit many + TX. */
8735         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
8736                                                              instruction_data,
8737                                                              n_instructions);
8738
8739         /* Mov all + validate. */
8740         n_instructions = instr_pattern_mov_all_validate_optimize(p,
8741                                                                  a,
8742                                                                  instructions,
8743                                                                  instruction_data,
8744                                                                  n_instructions);
8745
8746         /* DMA many. */
8747         n_instructions = instr_pattern_dma_many_optimize(instructions,
8748                                                          instruction_data,
8749                                                          n_instructions);
8750
8751         return n_instructions;
8752 }
8753
8754 static int
8755 instruction_config(struct rte_swx_pipeline *p,
8756                    struct action *a,
8757                    const char **instructions,
8758                    uint32_t n_instructions)
8759 {
8760         struct instruction *instr = NULL;
8761         struct instruction_data *data = NULL;
8762         int err = 0;
8763         uint32_t i;
8764
8765         CHECK(n_instructions, EINVAL);
8766         CHECK(instructions, EINVAL);
8767         for (i = 0; i < n_instructions; i++)
8768                 CHECK_INSTRUCTION(instructions[i], EINVAL);
8769
8770         /* Memory allocation. */
8771         instr = calloc(n_instructions, sizeof(struct instruction));
8772         if (!instr) {
8773                 err = -ENOMEM;
8774                 goto error;
8775         }
8776
8777         data = calloc(n_instructions, sizeof(struct instruction_data));
8778         if (!data) {
8779                 err = -ENOMEM;
8780                 goto error;
8781         }
8782
8783         for (i = 0; i < n_instructions; i++) {
8784                 char *string = strdup(instructions[i]);
8785                 if (!string) {
8786                         err = -ENOMEM;
8787                         goto error;
8788                 }
8789
8790                 err = instr_translate(p, a, string, &instr[i], &data[i]);
8791                 if (err) {
8792                         free(string);
8793                         goto error;
8794                 }
8795
8796                 free(string);
8797         }
8798
8799         err = instr_label_check(data, n_instructions);
8800         if (err)
8801                 goto error;
8802
8803         err = instr_verify(p, a, instr, data, n_instructions);
8804         if (err)
8805                 goto error;
8806
8807         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
8808
8809         err = instr_jmp_resolve(instr, data, n_instructions);
8810         if (err)
8811                 goto error;
8812
8813         if (a) {
8814                 a->instructions = instr;
8815                 a->n_instructions = n_instructions;
8816         } else {
8817                 p->instructions = instr;
8818                 p->n_instructions = n_instructions;
8819         }
8820
8821         free(data);
8822         return 0;
8823
8824 error:
8825         free(data);
8826         free(instr);
8827         return err;
8828 }
8829
8830 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
8831
8832 static instr_exec_t instruction_table[] = {
8833         [INSTR_RX] = instr_rx_exec,
8834         [INSTR_TX] = instr_tx_exec,
8835         [INSTR_TX_I] = instr_tx_i_exec,
8836
8837         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
8838         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
8839         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
8840         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
8841         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
8842         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
8843         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
8844         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
8845
8846         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
8847         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
8848         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
8849         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
8850         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
8851         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
8852         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
8853         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
8854         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
8855
8856         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
8857         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
8858
8859         [INSTR_MOV] = instr_mov_exec,
8860         [INSTR_MOV_MH] = instr_mov_mh_exec,
8861         [INSTR_MOV_HM] = instr_mov_hm_exec,
8862         [INSTR_MOV_HH] = instr_mov_hh_exec,
8863         [INSTR_MOV_I] = instr_mov_i_exec,
8864
8865         [INSTR_DMA_HT] = instr_dma_ht_exec,
8866         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
8867         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
8868         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
8869         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
8870         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
8871         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
8872         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
8873
8874         [INSTR_ALU_ADD] = instr_alu_add_exec,
8875         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
8876         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
8877         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
8878         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
8879         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
8880
8881         [INSTR_ALU_SUB] = instr_alu_sub_exec,
8882         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
8883         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
8884         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
8885         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
8886         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
8887
8888         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
8889         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
8890         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
8891         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
8892
8893         [INSTR_ALU_AND] = instr_alu_and_exec,
8894         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
8895         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
8896         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
8897         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
8898
8899         [INSTR_ALU_OR] = instr_alu_or_exec,
8900         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
8901         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
8902         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
8903         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
8904
8905         [INSTR_ALU_XOR] = instr_alu_xor_exec,
8906         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
8907         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
8908         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
8909         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
8910
8911         [INSTR_ALU_SHL] = instr_alu_shl_exec,
8912         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
8913         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
8914         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
8915         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
8916         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
8917
8918         [INSTR_ALU_SHR] = instr_alu_shr_exec,
8919         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
8920         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
8921         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
8922         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
8923         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
8924
8925         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
8926         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
8927         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
8928
8929         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
8930         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
8931         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
8932         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
8933         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
8934         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
8935
8936         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
8937         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
8938         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
8939         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
8940         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
8941         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
8942         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
8943         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
8944         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
8945
8946         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
8947         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
8948         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
8949         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
8950         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
8951         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
8952         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
8953         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
8954         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
8955
8956         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
8957         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
8958         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
8959
8960         [INSTR_METER_HHM] = instr_meter_hhm_exec,
8961         [INSTR_METER_HHI] = instr_meter_hhi_exec,
8962         [INSTR_METER_HMM] = instr_meter_hmm_exec,
8963         [INSTR_METER_HMI] = instr_meter_hmi_exec,
8964         [INSTR_METER_MHM] = instr_meter_mhm_exec,
8965         [INSTR_METER_MHI] = instr_meter_mhi_exec,
8966         [INSTR_METER_MMM] = instr_meter_mmm_exec,
8967         [INSTR_METER_MMI] = instr_meter_mmi_exec,
8968         [INSTR_METER_IHM] = instr_meter_ihm_exec,
8969         [INSTR_METER_IHI] = instr_meter_ihi_exec,
8970         [INSTR_METER_IMM] = instr_meter_imm_exec,
8971         [INSTR_METER_IMI] = instr_meter_imi_exec,
8972
8973         [INSTR_TABLE] = instr_table_exec,
8974         [INSTR_SELECTOR] = instr_selector_exec,
8975         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
8976         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
8977
8978         [INSTR_JMP] = instr_jmp_exec,
8979         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
8980         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
8981         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
8982         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
8983         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
8984         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
8985
8986         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
8987         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
8988         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
8989         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
8990         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
8991
8992         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
8993         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
8994         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
8995         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
8996         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
8997
8998         [INSTR_JMP_LT] = instr_jmp_lt_exec,
8999         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
9000         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
9001         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
9002         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
9003         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
9004
9005         [INSTR_JMP_GT] = instr_jmp_gt_exec,
9006         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
9007         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
9008         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
9009         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
9010         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
9011
9012         [INSTR_RETURN] = instr_return_exec,
9013 };
9014
9015 static inline void
9016 instr_exec(struct rte_swx_pipeline *p)
9017 {
9018         struct thread *t = &p->threads[p->thread_id];
9019         struct instruction *ip = t->ip;
9020         instr_exec_t instr = instruction_table[ip->type];
9021
9022         instr(p);
9023 }
9024
9025 /*
9026  * Action.
9027  */
9028 static struct action *
9029 action_find(struct rte_swx_pipeline *p, const char *name)
9030 {
9031         struct action *elem;
9032
9033         if (!name)
9034                 return NULL;
9035
9036         TAILQ_FOREACH(elem, &p->actions, node)
9037                 if (strcmp(elem->name, name) == 0)
9038                         return elem;
9039
9040         return NULL;
9041 }
9042
9043 static struct action *
9044 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9045 {
9046         struct action *action = NULL;
9047
9048         TAILQ_FOREACH(action, &p->actions, node)
9049                 if (action->id == id)
9050                         return action;
9051
9052         return NULL;
9053 }
9054
9055 static struct field *
9056 action_field_find(struct action *a, const char *name)
9057 {
9058         return a->st ? struct_type_field_find(a->st, name) : NULL;
9059 }
9060
9061 static struct field *
9062 action_field_parse(struct action *action, const char *name)
9063 {
9064         if (name[0] != 't' || name[1] != '.')
9065                 return NULL;
9066
9067         return action_field_find(action, &name[2]);
9068 }
9069
9070 int
9071 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
9072                                const char *name,
9073                                const char *args_struct_type_name,
9074                                const char **instructions,
9075                                uint32_t n_instructions)
9076 {
9077         struct struct_type *args_struct_type = NULL;
9078         struct action *a;
9079         int err;
9080
9081         CHECK(p, EINVAL);
9082
9083         CHECK_NAME(name, EINVAL);
9084         CHECK(!action_find(p, name), EEXIST);
9085
9086         if (args_struct_type_name) {
9087                 CHECK_NAME(args_struct_type_name, EINVAL);
9088                 args_struct_type = struct_type_find(p, args_struct_type_name);
9089                 CHECK(args_struct_type, EINVAL);
9090                 CHECK(!args_struct_type->var_size, EINVAL);
9091         }
9092
9093         /* Node allocation. */
9094         a = calloc(1, sizeof(struct action));
9095         CHECK(a, ENOMEM);
9096         if (args_struct_type) {
9097                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
9098                 if (!a->args_endianness) {
9099                         free(a);
9100                         CHECK(0, ENOMEM);
9101                 }
9102         }
9103
9104         /* Node initialization. */
9105         strcpy(a->name, name);
9106         a->st = args_struct_type;
9107         a->id = p->n_actions;
9108
9109         /* Instruction translation. */
9110         err = instruction_config(p, a, instructions, n_instructions);
9111         if (err) {
9112                 free(a->args_endianness);
9113                 free(a);
9114                 return err;
9115         }
9116
9117         /* Node add to tailq. */
9118         TAILQ_INSERT_TAIL(&p->actions, a, node);
9119         p->n_actions++;
9120
9121         return 0;
9122 }
9123
9124 static int
9125 action_build(struct rte_swx_pipeline *p)
9126 {
9127         struct action *action;
9128
9129         p->action_instructions = calloc(p->n_actions,
9130                                         sizeof(struct instruction *));
9131         CHECK(p->action_instructions, ENOMEM);
9132
9133         TAILQ_FOREACH(action, &p->actions, node)
9134                 p->action_instructions[action->id] = action->instructions;
9135
9136         return 0;
9137 }
9138
9139 static void
9140 action_build_free(struct rte_swx_pipeline *p)
9141 {
9142         free(p->action_instructions);
9143         p->action_instructions = NULL;
9144 }
9145
9146 static void
9147 action_free(struct rte_swx_pipeline *p)
9148 {
9149         action_build_free(p);
9150
9151         for ( ; ; ) {
9152                 struct action *action;
9153
9154                 action = TAILQ_FIRST(&p->actions);
9155                 if (!action)
9156                         break;
9157
9158                 TAILQ_REMOVE(&p->actions, action, node);
9159                 free(action->instructions);
9160                 free(action);
9161         }
9162 }
9163
9164 static uint32_t
9165 action_arg_src_mov_count(struct action *a,
9166                          uint32_t arg_id,
9167                          struct instruction *instructions,
9168                          struct instruction_data *instruction_data,
9169                          uint32_t n_instructions)
9170 {
9171         uint32_t offset, n_users = 0, i;
9172
9173         if (!a->st ||
9174             (arg_id >= a->st->n_fields) ||
9175             !instructions ||
9176             !instruction_data ||
9177             !n_instructions)
9178                 return 0;
9179
9180         offset = a->st->fields[arg_id].offset / 8;
9181
9182         for (i = 0; i < n_instructions; i++) {
9183                 struct instruction *instr = &instructions[i];
9184                 struct instruction_data *data = &instruction_data[i];
9185
9186                 if (data->invalid ||
9187                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
9188                     instr->mov.src.struct_id ||
9189                     (instr->mov.src.offset != offset))
9190                         continue;
9191
9192                 n_users++;
9193         }
9194
9195         return n_users;
9196 }
9197
9198 /*
9199  * Table.
9200  */
9201 static struct table_type *
9202 table_type_find(struct rte_swx_pipeline *p, const char *name)
9203 {
9204         struct table_type *elem;
9205
9206         TAILQ_FOREACH(elem, &p->table_types, node)
9207                 if (strcmp(elem->name, name) == 0)
9208                         return elem;
9209
9210         return NULL;
9211 }
9212
9213 static struct table_type *
9214 table_type_resolve(struct rte_swx_pipeline *p,
9215                    const char *recommended_type_name,
9216                    enum rte_swx_table_match_type match_type)
9217 {
9218         struct table_type *elem;
9219
9220         /* Only consider the recommended type if the match type is correct. */
9221         if (recommended_type_name)
9222                 TAILQ_FOREACH(elem, &p->table_types, node)
9223                         if (!strcmp(elem->name, recommended_type_name) &&
9224                             (elem->match_type == match_type))
9225                                 return elem;
9226
9227         /* Ignore the recommended type and get the first element with this match
9228          * type.
9229          */
9230         TAILQ_FOREACH(elem, &p->table_types, node)
9231                 if (elem->match_type == match_type)
9232                         return elem;
9233
9234         return NULL;
9235 }
9236
9237 static struct table *
9238 table_find(struct rte_swx_pipeline *p, const char *name)
9239 {
9240         struct table *elem;
9241
9242         TAILQ_FOREACH(elem, &p->tables, node)
9243                 if (strcmp(elem->name, name) == 0)
9244                         return elem;
9245
9246         return NULL;
9247 }
9248
9249 static struct table *
9250 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9251 {
9252         struct table *table = NULL;
9253
9254         TAILQ_FOREACH(table, &p->tables, node)
9255                 if (table->id == id)
9256                         return table;
9257
9258         return NULL;
9259 }
9260
9261 int
9262 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
9263                                      const char *name,
9264                                      enum rte_swx_table_match_type match_type,
9265                                      struct rte_swx_table_ops *ops)
9266 {
9267         struct table_type *elem;
9268
9269         CHECK(p, EINVAL);
9270
9271         CHECK_NAME(name, EINVAL);
9272         CHECK(!table_type_find(p, name), EEXIST);
9273
9274         CHECK(ops, EINVAL);
9275         CHECK(ops->create, EINVAL);
9276         CHECK(ops->lkp, EINVAL);
9277         CHECK(ops->free, EINVAL);
9278
9279         /* Node allocation. */
9280         elem = calloc(1, sizeof(struct table_type));
9281         CHECK(elem, ENOMEM);
9282
9283         /* Node initialization. */
9284         strcpy(elem->name, name);
9285         elem->match_type = match_type;
9286         memcpy(&elem->ops, ops, sizeof(*ops));
9287
9288         /* Node add to tailq. */
9289         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
9290
9291         return 0;
9292 }
9293
9294 static int
9295 table_match_type_resolve(struct rte_swx_match_field_params *fields,
9296                          uint32_t n_fields,
9297                          enum rte_swx_table_match_type *match_type)
9298 {
9299         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
9300
9301         for (i = 0; i < n_fields; i++) {
9302                 struct rte_swx_match_field_params  *f = &fields[i];
9303
9304                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
9305                         n_fields_em++;
9306
9307                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
9308                         n_fields_lpm++;
9309         }
9310
9311         if ((n_fields_lpm > 1) ||
9312             (n_fields_lpm && (n_fields_em != n_fields - 1)))
9313                 return -EINVAL;
9314
9315         *match_type = (n_fields_em == n_fields) ?
9316                        RTE_SWX_TABLE_MATCH_EXACT :
9317                        RTE_SWX_TABLE_MATCH_WILDCARD;
9318
9319         return 0;
9320 }
9321
9322 static int
9323 table_match_fields_check(struct rte_swx_pipeline *p,
9324                          struct rte_swx_pipeline_table_params *params,
9325                          struct header **header)
9326 {
9327         struct header *h0 = NULL;
9328         struct field *hf, *mf;
9329         uint32_t *offset = NULL, i;
9330         int status = 0;
9331
9332         /* Return if no match fields. */
9333         if (!params->n_fields) {
9334                 if (params->fields) {
9335                         status = -EINVAL;
9336                         goto end;
9337                 }
9338
9339                 if (header)
9340                         *header = NULL;
9341
9342                 return 0;
9343         }
9344
9345         /* Memory allocation. */
9346         offset = calloc(params->n_fields, sizeof(uint32_t));
9347         if (!offset) {
9348                 status = -ENOMEM;
9349                 goto end;
9350         }
9351
9352         /* Check that all the match fields belong to either the same header or
9353          * to the meta-data.
9354          */
9355         hf = header_field_parse(p, params->fields[0].name, &h0);
9356         mf = metadata_field_parse(p, params->fields[0].name);
9357         if ((!hf && !mf) || (hf && hf->var_size)) {
9358                 status = -EINVAL;
9359                 goto end;
9360         }
9361
9362         offset[0] = h0 ? hf->offset : mf->offset;
9363
9364         for (i = 1; i < params->n_fields; i++)
9365                 if (h0) {
9366                         struct header *h;
9367
9368                         hf = header_field_parse(p, params->fields[i].name, &h);
9369                         if (!hf || (h->id != h0->id) || hf->var_size) {
9370                                 status = -EINVAL;
9371                                 goto end;
9372                         }
9373
9374                         offset[i] = hf->offset;
9375                 } else {
9376                         mf = metadata_field_parse(p, params->fields[i].name);
9377                         if (!mf) {
9378                                 status = -EINVAL;
9379                                 goto end;
9380                         }
9381
9382                         offset[i] = mf->offset;
9383                 }
9384
9385         /* Check that there are no duplicated match fields. */
9386         for (i = 0; i < params->n_fields; i++) {
9387                 uint32_t j;
9388
9389                 for (j = 0; j < i; j++)
9390                         if (offset[j] == offset[i]) {
9391                                 status = -EINVAL;
9392                                 goto end;
9393                         }
9394         }
9395
9396         /* Return. */
9397         if (header)
9398                 *header = h0;
9399
9400 end:
9401         free(offset);
9402         return status;
9403 }
9404
9405 int
9406 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
9407                               const char *name,
9408                               struct rte_swx_pipeline_table_params *params,
9409                               const char *recommended_table_type_name,
9410                               const char *args,
9411                               uint32_t size)
9412 {
9413         struct table_type *type;
9414         struct table *t;
9415         struct action *default_action;
9416         struct header *header = NULL;
9417         uint32_t action_data_size_max = 0, i;
9418         int status = 0;
9419
9420         CHECK(p, EINVAL);
9421
9422         CHECK_NAME(name, EINVAL);
9423         CHECK(!table_find(p, name), EEXIST);
9424         CHECK(!selector_find(p, name), EEXIST);
9425
9426         CHECK(params, EINVAL);
9427
9428         /* Match checks. */
9429         status = table_match_fields_check(p, params, &header);
9430         if (status)
9431                 return status;
9432
9433         /* Action checks. */
9434         CHECK(params->n_actions, EINVAL);
9435         CHECK(params->action_names, EINVAL);
9436         for (i = 0; i < params->n_actions; i++) {
9437                 const char *action_name = params->action_names[i];
9438                 struct action *a;
9439                 uint32_t action_data_size;
9440
9441                 CHECK_NAME(action_name, EINVAL);
9442
9443                 a = action_find(p, action_name);
9444                 CHECK(a, EINVAL);
9445
9446                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
9447                 if (action_data_size > action_data_size_max)
9448                         action_data_size_max = action_data_size;
9449         }
9450
9451         CHECK_NAME(params->default_action_name, EINVAL);
9452         for (i = 0; i < p->n_actions; i++)
9453                 if (!strcmp(params->action_names[i],
9454                             params->default_action_name))
9455                         break;
9456         CHECK(i < params->n_actions, EINVAL);
9457         default_action = action_find(p, params->default_action_name);
9458         CHECK((default_action->st && params->default_action_data) ||
9459               !params->default_action_data, EINVAL);
9460
9461         /* Table type checks. */
9462         if (recommended_table_type_name)
9463                 CHECK_NAME(recommended_table_type_name, EINVAL);
9464
9465         if (params->n_fields) {
9466                 enum rte_swx_table_match_type match_type;
9467
9468                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
9469                 if (status)
9470                         return status;
9471
9472                 type = table_type_resolve(p, recommended_table_type_name, match_type);
9473                 CHECK(type, EINVAL);
9474         } else {
9475                 type = NULL;
9476         }
9477
9478         /* Memory allocation. */
9479         t = calloc(1, sizeof(struct table));
9480         CHECK(t, ENOMEM);
9481
9482         t->fields = calloc(params->n_fields, sizeof(struct match_field));
9483         if (!t->fields) {
9484                 free(t);
9485                 CHECK(0, ENOMEM);
9486         }
9487
9488         t->actions = calloc(params->n_actions, sizeof(struct action *));
9489         if (!t->actions) {
9490                 free(t->fields);
9491                 free(t);
9492                 CHECK(0, ENOMEM);
9493         }
9494
9495         if (action_data_size_max) {
9496                 t->default_action_data = calloc(1, action_data_size_max);
9497                 if (!t->default_action_data) {
9498                         free(t->actions);
9499                         free(t->fields);
9500                         free(t);
9501                         CHECK(0, ENOMEM);
9502                 }
9503         }
9504
9505         /* Node initialization. */
9506         strcpy(t->name, name);
9507         if (args && args[0])
9508                 strcpy(t->args, args);
9509         t->type = type;
9510
9511         for (i = 0; i < params->n_fields; i++) {
9512                 struct rte_swx_match_field_params *field = &params->fields[i];
9513                 struct match_field *f = &t->fields[i];
9514
9515                 f->match_type = field->match_type;
9516                 f->field = header ?
9517                         header_field_parse(p, field->name, NULL) :
9518                         metadata_field_parse(p, field->name);
9519         }
9520         t->n_fields = params->n_fields;
9521         t->header = header;
9522
9523         for (i = 0; i < params->n_actions; i++)
9524                 t->actions[i] = action_find(p, params->action_names[i]);
9525         t->default_action = default_action;
9526         if (default_action->st)
9527                 memcpy(t->default_action_data,
9528                        params->default_action_data,
9529                        default_action->st->n_bits / 8);
9530         t->n_actions = params->n_actions;
9531         t->default_action_is_const = params->default_action_is_const;
9532         t->action_data_size_max = action_data_size_max;
9533
9534         t->size = size;
9535         t->id = p->n_tables;
9536
9537         /* Node add to tailq. */
9538         TAILQ_INSERT_TAIL(&p->tables, t, node);
9539         p->n_tables++;
9540
9541         return 0;
9542 }
9543
9544 static struct rte_swx_table_params *
9545 table_params_get(struct table *table)
9546 {
9547         struct rte_swx_table_params *params;
9548         struct field *first, *last;
9549         uint8_t *key_mask;
9550         uint32_t key_size, key_offset, action_data_size, i;
9551
9552         /* Memory allocation. */
9553         params = calloc(1, sizeof(struct rte_swx_table_params));
9554         if (!params)
9555                 return NULL;
9556
9557         /* Find first (smallest offset) and last (biggest offset) match fields. */
9558         first = table->fields[0].field;
9559         last = table->fields[0].field;
9560
9561         for (i = 0; i < table->n_fields; i++) {
9562                 struct field *f = table->fields[i].field;
9563
9564                 if (f->offset < first->offset)
9565                         first = f;
9566
9567                 if (f->offset > last->offset)
9568                         last = f;
9569         }
9570
9571         /* Key offset and size. */
9572         key_offset = first->offset / 8;
9573         key_size = (last->offset + last->n_bits - first->offset) / 8;
9574
9575         /* Memory allocation. */
9576         key_mask = calloc(1, key_size);
9577         if (!key_mask) {
9578                 free(params);
9579                 return NULL;
9580         }
9581
9582         /* Key mask. */
9583         for (i = 0; i < table->n_fields; i++) {
9584                 struct field *f = table->fields[i].field;
9585                 uint32_t start = (f->offset - first->offset) / 8;
9586                 size_t size = f->n_bits / 8;
9587
9588                 memset(&key_mask[start], 0xFF, size);
9589         }
9590
9591         /* Action data size. */
9592         action_data_size = 0;
9593         for (i = 0; i < table->n_actions; i++) {
9594                 struct action *action = table->actions[i];
9595                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
9596
9597                 if (ads > action_data_size)
9598                         action_data_size = ads;
9599         }
9600
9601         /* Fill in. */
9602         params->match_type = table->type->match_type;
9603         params->key_size = key_size;
9604         params->key_offset = key_offset;
9605         params->key_mask0 = key_mask;
9606         params->action_data_size = action_data_size;
9607         params->n_keys_max = table->size;
9608
9609         return params;
9610 }
9611
9612 static void
9613 table_params_free(struct rte_swx_table_params *params)
9614 {
9615         if (!params)
9616                 return;
9617
9618         free(params->key_mask0);
9619         free(params);
9620 }
9621
9622 static int
9623 table_stub_lkp(void *table __rte_unused,
9624                void *mailbox __rte_unused,
9625                uint8_t **key __rte_unused,
9626                uint64_t *action_id __rte_unused,
9627                uint8_t **action_data __rte_unused,
9628                int *hit)
9629 {
9630         *hit = 0;
9631         return 1; /* DONE. */
9632 }
9633
9634 static int
9635 table_build(struct rte_swx_pipeline *p)
9636 {
9637         uint32_t i;
9638
9639         /* Per pipeline: table statistics. */
9640         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
9641         CHECK(p->table_stats, ENOMEM);
9642
9643         for (i = 0; i < p->n_tables; i++) {
9644                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9645                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
9646         }
9647
9648         /* Per thread: table runt-time. */
9649         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9650                 struct thread *t = &p->threads[i];
9651                 struct table *table;
9652
9653                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
9654                 CHECK(t->tables, ENOMEM);
9655
9656                 TAILQ_FOREACH(table, &p->tables, node) {
9657                         struct table_runtime *r = &t->tables[table->id];
9658
9659                         if (table->type) {
9660                                 uint64_t size;
9661
9662                                 size = table->type->ops.mailbox_size_get();
9663
9664                                 /* r->func. */
9665                                 r->func = table->type->ops.lkp;
9666
9667                                 /* r->mailbox. */
9668                                 if (size) {
9669                                         r->mailbox = calloc(1, size);
9670                                         CHECK(r->mailbox, ENOMEM);
9671                                 }
9672
9673                                 /* r->key. */
9674                                 r->key = table->header ?
9675                                         &t->structs[table->header->struct_id] :
9676                                         &t->structs[p->metadata_struct_id];
9677                         } else {
9678                                 r->func = table_stub_lkp;
9679                         }
9680                 }
9681         }
9682
9683         return 0;
9684 }
9685
9686 static void
9687 table_build_free(struct rte_swx_pipeline *p)
9688 {
9689         uint32_t i;
9690
9691         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9692                 struct thread *t = &p->threads[i];
9693                 uint32_t j;
9694
9695                 if (!t->tables)
9696                         continue;
9697
9698                 for (j = 0; j < p->n_tables; j++) {
9699                         struct table_runtime *r = &t->tables[j];
9700
9701                         free(r->mailbox);
9702                 }
9703
9704                 free(t->tables);
9705                 t->tables = NULL;
9706         }
9707
9708         if (p->table_stats) {
9709                 for (i = 0; i < p->n_tables; i++)
9710                         free(p->table_stats[i].n_pkts_action);
9711
9712                 free(p->table_stats);
9713         }
9714 }
9715
9716 static void
9717 table_free(struct rte_swx_pipeline *p)
9718 {
9719         table_build_free(p);
9720
9721         /* Tables. */
9722         for ( ; ; ) {
9723                 struct table *elem;
9724
9725                 elem = TAILQ_FIRST(&p->tables);
9726                 if (!elem)
9727                         break;
9728
9729                 TAILQ_REMOVE(&p->tables, elem, node);
9730                 free(elem->fields);
9731                 free(elem->actions);
9732                 free(elem->default_action_data);
9733                 free(elem);
9734         }
9735
9736         /* Table types. */
9737         for ( ; ; ) {
9738                 struct table_type *elem;
9739
9740                 elem = TAILQ_FIRST(&p->table_types);
9741                 if (!elem)
9742                         break;
9743
9744                 TAILQ_REMOVE(&p->table_types, elem, node);
9745                 free(elem);
9746         }
9747 }
9748
9749 /*
9750  * Selector.
9751  */
9752 static struct selector *
9753 selector_find(struct rte_swx_pipeline *p, const char *name)
9754 {
9755         struct selector *s;
9756
9757         TAILQ_FOREACH(s, &p->selectors, node)
9758                 if (strcmp(s->name, name) == 0)
9759                         return s;
9760
9761         return NULL;
9762 }
9763
9764 static struct selector *
9765 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9766 {
9767         struct selector *s = NULL;
9768
9769         TAILQ_FOREACH(s, &p->selectors, node)
9770                 if (s->id == id)
9771                         return s;
9772
9773         return NULL;
9774 }
9775
9776 static int
9777 selector_fields_check(struct rte_swx_pipeline *p,
9778                       struct rte_swx_pipeline_selector_params *params,
9779                       struct header **header)
9780 {
9781         struct header *h0 = NULL;
9782         struct field *hf, *mf;
9783         uint32_t i;
9784
9785         /* Return if no selector fields. */
9786         if (!params->n_selector_fields || !params->selector_field_names)
9787                 return -EINVAL;
9788
9789         /* Check that all the selector fields either belong to the same header
9790          * or are all meta-data fields.
9791          */
9792         hf = header_field_parse(p, params->selector_field_names[0], &h0);
9793         mf = metadata_field_parse(p, params->selector_field_names[0]);
9794         if (!hf && !mf)
9795                 return -EINVAL;
9796
9797         for (i = 1; i < params->n_selector_fields; i++)
9798                 if (h0) {
9799                         struct header *h;
9800
9801                         hf = header_field_parse(p, params->selector_field_names[i], &h);
9802                         if (!hf || (h->id != h0->id))
9803                                 return -EINVAL;
9804                 } else {
9805                         mf = metadata_field_parse(p, params->selector_field_names[i]);
9806                         if (!mf)
9807                                 return -EINVAL;
9808                 }
9809
9810         /* Check that there are no duplicated match fields. */
9811         for (i = 0; i < params->n_selector_fields; i++) {
9812                 const char *field_name = params->selector_field_names[i];
9813                 uint32_t j;
9814
9815                 for (j = i + 1; j < params->n_selector_fields; j++)
9816                         if (!strcmp(params->selector_field_names[j], field_name))
9817                                 return -EINVAL;
9818         }
9819
9820         /* Return. */
9821         if (header)
9822                 *header = h0;
9823
9824         return 0;
9825 }
9826
9827 int
9828 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
9829                                  const char *name,
9830                                  struct rte_swx_pipeline_selector_params *params)
9831 {
9832         struct selector *s;
9833         struct header *selector_header = NULL;
9834         struct field *group_id_field, *member_id_field;
9835         uint32_t i;
9836         int status = 0;
9837
9838         CHECK(p, EINVAL);
9839
9840         CHECK_NAME(name, EINVAL);
9841         CHECK(!table_find(p, name), EEXIST);
9842         CHECK(!selector_find(p, name), EEXIST);
9843
9844         CHECK(params, EINVAL);
9845
9846         CHECK_NAME(params->group_id_field_name, EINVAL);
9847         group_id_field = metadata_field_parse(p, params->group_id_field_name);
9848         CHECK(group_id_field, EINVAL);
9849
9850         for (i = 0; i < params->n_selector_fields; i++) {
9851                 const char *field_name = params->selector_field_names[i];
9852
9853                 CHECK_NAME(field_name, EINVAL);
9854         }
9855         status = selector_fields_check(p, params, &selector_header);
9856         if (status)
9857                 return status;
9858
9859         CHECK_NAME(params->member_id_field_name, EINVAL);
9860         member_id_field = metadata_field_parse(p, params->member_id_field_name);
9861         CHECK(member_id_field, EINVAL);
9862
9863         CHECK(params->n_groups_max, EINVAL);
9864
9865         CHECK(params->n_members_per_group_max, EINVAL);
9866
9867         /* Memory allocation. */
9868         s = calloc(1, sizeof(struct selector));
9869         if (!s) {
9870                 status = -ENOMEM;
9871                 goto error;
9872         }
9873
9874         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
9875         if (!s->selector_fields) {
9876                 status = -ENOMEM;
9877                 goto error;
9878         }
9879
9880         /* Node initialization. */
9881         strcpy(s->name, name);
9882
9883         s->group_id_field = group_id_field;
9884
9885         for (i = 0; i < params->n_selector_fields; i++) {
9886                 const char *field_name = params->selector_field_names[i];
9887
9888                 s->selector_fields[i] = selector_header ?
9889                         header_field_parse(p, field_name, NULL) :
9890                         metadata_field_parse(p, field_name);
9891         }
9892
9893         s->n_selector_fields = params->n_selector_fields;
9894
9895         s->selector_header = selector_header;
9896
9897         s->member_id_field = member_id_field;
9898
9899         s->n_groups_max = params->n_groups_max;
9900
9901         s->n_members_per_group_max = params->n_members_per_group_max;
9902
9903         s->id = p->n_selectors;
9904
9905         /* Node add to tailq. */
9906         TAILQ_INSERT_TAIL(&p->selectors, s, node);
9907         p->n_selectors++;
9908
9909         return 0;
9910
9911 error:
9912         if (!s)
9913                 return status;
9914
9915         free(s->selector_fields);
9916
9917         free(s);
9918
9919         return status;
9920 }
9921
9922 static void
9923 selector_params_free(struct rte_swx_table_selector_params *params)
9924 {
9925         if (!params)
9926                 return;
9927
9928         free(params->selector_mask);
9929
9930         free(params);
9931 }
9932
9933 static struct rte_swx_table_selector_params *
9934 selector_table_params_get(struct selector *s)
9935 {
9936         struct rte_swx_table_selector_params *params = NULL;
9937         struct field *first, *last;
9938         uint32_t i;
9939
9940         /* Memory allocation. */
9941         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
9942         if (!params)
9943                 goto error;
9944
9945         /* Group ID. */
9946         params->group_id_offset = s->group_id_field->offset / 8;
9947
9948         /* Find first (smallest offset) and last (biggest offset) selector fields. */
9949         first = s->selector_fields[0];
9950         last = s->selector_fields[0];
9951
9952         for (i = 0; i < s->n_selector_fields; i++) {
9953                 struct field *f = s->selector_fields[i];
9954
9955                 if (f->offset < first->offset)
9956                         first = f;
9957
9958                 if (f->offset > last->offset)
9959                         last = f;
9960         }
9961
9962         /* Selector offset and size. */
9963         params->selector_offset = first->offset / 8;
9964         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
9965
9966         /* Memory allocation. */
9967         params->selector_mask = calloc(1, params->selector_size);
9968         if (!params->selector_mask)
9969                 goto error;
9970
9971         /* Selector mask. */
9972         for (i = 0; i < s->n_selector_fields; i++) {
9973                 struct field *f = s->selector_fields[i];
9974                 uint32_t start = (f->offset - first->offset) / 8;
9975                 size_t size = f->n_bits / 8;
9976
9977                 memset(&params->selector_mask[start], 0xFF, size);
9978         }
9979
9980         /* Member ID. */
9981         params->member_id_offset = s->member_id_field->offset / 8;
9982
9983         /* Maximum number of groups. */
9984         params->n_groups_max = s->n_groups_max;
9985
9986         /* Maximum number of members per group. */
9987         params->n_members_per_group_max = s->n_members_per_group_max;
9988
9989         return params;
9990
9991 error:
9992         selector_params_free(params);
9993         return NULL;
9994 }
9995
9996 static void
9997 selector_build_free(struct rte_swx_pipeline *p)
9998 {
9999         uint32_t i;
10000
10001         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10002                 struct thread *t = &p->threads[i];
10003                 uint32_t j;
10004
10005                 if (!t->selectors)
10006                         continue;
10007
10008                 for (j = 0; j < p->n_selectors; j++) {
10009                         struct selector_runtime *r = &t->selectors[j];
10010
10011                         free(r->mailbox);
10012                 }
10013
10014                 free(t->selectors);
10015                 t->selectors = NULL;
10016         }
10017
10018         free(p->selector_stats);
10019         p->selector_stats = NULL;
10020 }
10021
10022 static int
10023 selector_build(struct rte_swx_pipeline *p)
10024 {
10025         uint32_t i;
10026         int status = 0;
10027
10028         /* Per pipeline: selector statistics. */
10029         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
10030         if (!p->selector_stats) {
10031                 status = -ENOMEM;
10032                 goto error;
10033         }
10034
10035         /* Per thread: selector run-time. */
10036         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10037                 struct thread *t = &p->threads[i];
10038                 struct selector *s;
10039
10040                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
10041                 if (!t->selectors) {
10042                         status = -ENOMEM;
10043                         goto error;
10044                 }
10045
10046                 TAILQ_FOREACH(s, &p->selectors, node) {
10047                         struct selector_runtime *r = &t->selectors[s->id];
10048                         uint64_t size;
10049
10050                         /* r->mailbox. */
10051                         size = rte_swx_table_selector_mailbox_size_get();
10052                         if (size) {
10053                                 r->mailbox = calloc(1, size);
10054                                 if (!r->mailbox) {
10055                                         status = -ENOMEM;
10056                                         goto error;
10057                                 }
10058                         }
10059
10060                         /* r->group_id_buffer. */
10061                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
10062
10063                         /* r->selector_buffer. */
10064                         r->selector_buffer = s->selector_header ?
10065                                 &t->structs[s->selector_header->struct_id] :
10066                                 &t->structs[p->metadata_struct_id];
10067
10068                         /* r->member_id_buffer. */
10069                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
10070                 }
10071         }
10072
10073         return 0;
10074
10075 error:
10076         selector_build_free(p);
10077         return status;
10078 }
10079
10080 static void
10081 selector_free(struct rte_swx_pipeline *p)
10082 {
10083         selector_build_free(p);
10084
10085         /* Selector tables. */
10086         for ( ; ; ) {
10087                 struct selector *elem;
10088
10089                 elem = TAILQ_FIRST(&p->selectors);
10090                 if (!elem)
10091                         break;
10092
10093                 TAILQ_REMOVE(&p->selectors, elem, node);
10094                 free(elem->selector_fields);
10095                 free(elem);
10096         }
10097 }
10098
10099 /*
10100  * Table state.
10101  */
10102 static int
10103 table_state_build(struct rte_swx_pipeline *p)
10104 {
10105         struct table *table;
10106         struct selector *s;
10107
10108         p->table_state = calloc(p->n_tables + p->n_selectors,
10109                                 sizeof(struct rte_swx_table_state));
10110         CHECK(p->table_state, ENOMEM);
10111
10112         TAILQ_FOREACH(table, &p->tables, node) {
10113                 struct rte_swx_table_state *ts = &p->table_state[table->id];
10114
10115                 if (table->type) {
10116                         struct rte_swx_table_params *params;
10117
10118                         /* ts->obj. */
10119                         params = table_params_get(table);
10120                         CHECK(params, ENOMEM);
10121
10122                         ts->obj = table->type->ops.create(params,
10123                                 NULL,
10124                                 table->args,
10125                                 p->numa_node);
10126
10127                         table_params_free(params);
10128                         CHECK(ts->obj, ENODEV);
10129                 }
10130
10131                 /* ts->default_action_data. */
10132                 if (table->action_data_size_max) {
10133                         ts->default_action_data =
10134                                 malloc(table->action_data_size_max);
10135                         CHECK(ts->default_action_data, ENOMEM);
10136
10137                         memcpy(ts->default_action_data,
10138                                table->default_action_data,
10139                                table->action_data_size_max);
10140                 }
10141
10142                 /* ts->default_action_id. */
10143                 ts->default_action_id = table->default_action->id;
10144         }
10145
10146         TAILQ_FOREACH(s, &p->selectors, node) {
10147                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
10148                 struct rte_swx_table_selector_params *params;
10149
10150                 /* ts->obj. */
10151                 params = selector_table_params_get(s);
10152                 CHECK(params, ENOMEM);
10153
10154                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
10155
10156                 selector_params_free(params);
10157                 CHECK(ts->obj, ENODEV);
10158         }
10159
10160         return 0;
10161 }
10162
10163 static void
10164 table_state_build_free(struct rte_swx_pipeline *p)
10165 {
10166         uint32_t i;
10167
10168         if (!p->table_state)
10169                 return;
10170
10171         for (i = 0; i < p->n_tables; i++) {
10172                 struct rte_swx_table_state *ts = &p->table_state[i];
10173                 struct table *table = table_find_by_id(p, i);
10174
10175                 /* ts->obj. */
10176                 if (table->type && ts->obj)
10177                         table->type->ops.free(ts->obj);
10178
10179                 /* ts->default_action_data. */
10180                 free(ts->default_action_data);
10181         }
10182
10183         for (i = 0; i < p->n_selectors; i++) {
10184                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
10185
10186                 /* ts->obj. */
10187                 if (ts->obj)
10188                         rte_swx_table_selector_free(ts->obj);
10189         }
10190
10191         free(p->table_state);
10192         p->table_state = NULL;
10193 }
10194
10195 static void
10196 table_state_free(struct rte_swx_pipeline *p)
10197 {
10198         table_state_build_free(p);
10199 }
10200
10201 /*
10202  * Register array.
10203  */
10204 static struct regarray *
10205 regarray_find(struct rte_swx_pipeline *p, const char *name)
10206 {
10207         struct regarray *elem;
10208
10209         TAILQ_FOREACH(elem, &p->regarrays, node)
10210                 if (!strcmp(elem->name, name))
10211                         return elem;
10212
10213         return NULL;
10214 }
10215
10216 static struct regarray *
10217 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
10218 {
10219         struct regarray *elem = NULL;
10220
10221         TAILQ_FOREACH(elem, &p->regarrays, node)
10222                 if (elem->id == id)
10223                         return elem;
10224
10225         return NULL;
10226 }
10227
10228 int
10229 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
10230                               const char *name,
10231                               uint32_t size,
10232                               uint64_t init_val)
10233 {
10234         struct regarray *r;
10235
10236         CHECK(p, EINVAL);
10237
10238         CHECK_NAME(name, EINVAL);
10239         CHECK(!regarray_find(p, name), EEXIST);
10240
10241         CHECK(size, EINVAL);
10242         size = rte_align32pow2(size);
10243
10244         /* Memory allocation. */
10245         r = calloc(1, sizeof(struct regarray));
10246         CHECK(r, ENOMEM);
10247
10248         /* Node initialization. */
10249         strcpy(r->name, name);
10250         r->init_val = init_val;
10251         r->size = size;
10252         r->id = p->n_regarrays;
10253
10254         /* Node add to tailq. */
10255         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
10256         p->n_regarrays++;
10257
10258         return 0;
10259 }
10260
10261 static int
10262 regarray_build(struct rte_swx_pipeline *p)
10263 {
10264         struct regarray *regarray;
10265
10266         if (!p->n_regarrays)
10267                 return 0;
10268
10269         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
10270         CHECK(p->regarray_runtime, ENOMEM);
10271
10272         TAILQ_FOREACH(regarray, &p->regarrays, node) {
10273                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
10274                 uint32_t i;
10275
10276                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
10277                                          RTE_CACHE_LINE_SIZE,
10278                                          p->numa_node);
10279                 CHECK(r->regarray, ENOMEM);
10280
10281                 if (regarray->init_val)
10282                         for (i = 0; i < regarray->size; i++)
10283                                 r->regarray[i] = regarray->init_val;
10284
10285                 r->size_mask = regarray->size - 1;
10286         }
10287
10288         return 0;
10289 }
10290
10291 static void
10292 regarray_build_free(struct rte_swx_pipeline *p)
10293 {
10294         uint32_t i;
10295
10296         if (!p->regarray_runtime)
10297                 return;
10298
10299         for (i = 0; i < p->n_regarrays; i++) {
10300                 struct regarray *regarray = regarray_find_by_id(p, i);
10301                 struct regarray_runtime *r = &p->regarray_runtime[i];
10302
10303                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
10304         }
10305
10306         free(p->regarray_runtime);
10307         p->regarray_runtime = NULL;
10308 }
10309
10310 static void
10311 regarray_free(struct rte_swx_pipeline *p)
10312 {
10313         regarray_build_free(p);
10314
10315         for ( ; ; ) {
10316                 struct regarray *elem;
10317
10318                 elem = TAILQ_FIRST(&p->regarrays);
10319                 if (!elem)
10320                         break;
10321
10322                 TAILQ_REMOVE(&p->regarrays, elem, node);
10323                 free(elem);
10324         }
10325 }
10326
10327 /*
10328  * Meter array.
10329  */
10330 static struct meter_profile *
10331 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
10332 {
10333         struct meter_profile *elem;
10334
10335         TAILQ_FOREACH(elem, &p->meter_profiles, node)
10336                 if (!strcmp(elem->name, name))
10337                         return elem;
10338
10339         return NULL;
10340 }
10341
10342 static struct metarray *
10343 metarray_find(struct rte_swx_pipeline *p, const char *name)
10344 {
10345         struct metarray *elem;
10346
10347         TAILQ_FOREACH(elem, &p->metarrays, node)
10348                 if (!strcmp(elem->name, name))
10349                         return elem;
10350
10351         return NULL;
10352 }
10353
10354 static struct metarray *
10355 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
10356 {
10357         struct metarray *elem = NULL;
10358
10359         TAILQ_FOREACH(elem, &p->metarrays, node)
10360                 if (elem->id == id)
10361                         return elem;
10362
10363         return NULL;
10364 }
10365
10366 int
10367 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
10368                                  const char *name,
10369                                  uint32_t size)
10370 {
10371         struct metarray *m;
10372
10373         CHECK(p, EINVAL);
10374
10375         CHECK_NAME(name, EINVAL);
10376         CHECK(!metarray_find(p, name), EEXIST);
10377
10378         CHECK(size, EINVAL);
10379         size = rte_align32pow2(size);
10380
10381         /* Memory allocation. */
10382         m = calloc(1, sizeof(struct metarray));
10383         CHECK(m, ENOMEM);
10384
10385         /* Node initialization. */
10386         strcpy(m->name, name);
10387         m->size = size;
10388         m->id = p->n_metarrays;
10389
10390         /* Node add to tailq. */
10391         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
10392         p->n_metarrays++;
10393
10394         return 0;
10395 }
10396
10397 struct meter_profile meter_profile_default = {
10398         .node = {0},
10399         .name = "",
10400         .params = {0},
10401
10402         .profile = {
10403                 .cbs = 10000,
10404                 .pbs = 10000,
10405                 .cir_period = 1,
10406                 .cir_bytes_per_period = 1,
10407                 .pir_period = 1,
10408                 .pir_bytes_per_period = 1,
10409         },
10410
10411         .n_users = 0,
10412 };
10413
10414 static void
10415 meter_init(struct meter *m)
10416 {
10417         memset(m, 0, sizeof(struct meter));
10418         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
10419         m->profile = &meter_profile_default;
10420         m->color_mask = RTE_COLOR_GREEN;
10421
10422         meter_profile_default.n_users++;
10423 }
10424
10425 static int
10426 metarray_build(struct rte_swx_pipeline *p)
10427 {
10428         struct metarray *m;
10429
10430         if (!p->n_metarrays)
10431                 return 0;
10432
10433         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
10434         CHECK(p->metarray_runtime, ENOMEM);
10435
10436         TAILQ_FOREACH(m, &p->metarrays, node) {
10437                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
10438                 uint32_t i;
10439
10440                 r->metarray = env_malloc(m->size * sizeof(struct meter),
10441                                          RTE_CACHE_LINE_SIZE,
10442                                          p->numa_node);
10443                 CHECK(r->metarray, ENOMEM);
10444
10445                 for (i = 0; i < m->size; i++)
10446                         meter_init(&r->metarray[i]);
10447
10448                 r->size_mask = m->size - 1;
10449         }
10450
10451         return 0;
10452 }
10453
10454 static void
10455 metarray_build_free(struct rte_swx_pipeline *p)
10456 {
10457         uint32_t i;
10458
10459         if (!p->metarray_runtime)
10460                 return;
10461
10462         for (i = 0; i < p->n_metarrays; i++) {
10463                 struct metarray *m = metarray_find_by_id(p, i);
10464                 struct metarray_runtime *r = &p->metarray_runtime[i];
10465
10466                 env_free(r->metarray, m->size * sizeof(struct meter));
10467         }
10468
10469         free(p->metarray_runtime);
10470         p->metarray_runtime = NULL;
10471 }
10472
10473 static void
10474 metarray_free(struct rte_swx_pipeline *p)
10475 {
10476         metarray_build_free(p);
10477
10478         /* Meter arrays. */
10479         for ( ; ; ) {
10480                 struct metarray *elem;
10481
10482                 elem = TAILQ_FIRST(&p->metarrays);
10483                 if (!elem)
10484                         break;
10485
10486                 TAILQ_REMOVE(&p->metarrays, elem, node);
10487                 free(elem);
10488         }
10489
10490         /* Meter profiles. */
10491         for ( ; ; ) {
10492                 struct meter_profile *elem;
10493
10494                 elem = TAILQ_FIRST(&p->meter_profiles);
10495                 if (!elem)
10496                         break;
10497
10498                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
10499                 free(elem);
10500         }
10501 }
10502
10503 /*
10504  * Pipeline.
10505  */
10506 int
10507 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
10508 {
10509         struct rte_swx_pipeline *pipeline;
10510
10511         /* Check input parameters. */
10512         CHECK(p, EINVAL);
10513
10514         /* Memory allocation. */
10515         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
10516         CHECK(pipeline, ENOMEM);
10517
10518         /* Initialization. */
10519         TAILQ_INIT(&pipeline->struct_types);
10520         TAILQ_INIT(&pipeline->port_in_types);
10521         TAILQ_INIT(&pipeline->ports_in);
10522         TAILQ_INIT(&pipeline->port_out_types);
10523         TAILQ_INIT(&pipeline->ports_out);
10524         TAILQ_INIT(&pipeline->extern_types);
10525         TAILQ_INIT(&pipeline->extern_objs);
10526         TAILQ_INIT(&pipeline->extern_funcs);
10527         TAILQ_INIT(&pipeline->headers);
10528         TAILQ_INIT(&pipeline->actions);
10529         TAILQ_INIT(&pipeline->table_types);
10530         TAILQ_INIT(&pipeline->tables);
10531         TAILQ_INIT(&pipeline->selectors);
10532         TAILQ_INIT(&pipeline->regarrays);
10533         TAILQ_INIT(&pipeline->meter_profiles);
10534         TAILQ_INIT(&pipeline->metarrays);
10535
10536         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
10537         pipeline->numa_node = numa_node;
10538
10539         *p = pipeline;
10540         return 0;
10541 }
10542
10543 void
10544 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
10545 {
10546         if (!p)
10547                 return;
10548
10549         free(p->instructions);
10550
10551         metarray_free(p);
10552         regarray_free(p);
10553         table_state_free(p);
10554         selector_free(p);
10555         table_free(p);
10556         action_free(p);
10557         metadata_free(p);
10558         header_free(p);
10559         extern_func_free(p);
10560         extern_obj_free(p);
10561         port_out_free(p);
10562         port_in_free(p);
10563         struct_free(p);
10564
10565         free(p);
10566 }
10567
10568 int
10569 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
10570                                      const char **instructions,
10571                                      uint32_t n_instructions)
10572 {
10573         int err;
10574         uint32_t i;
10575
10576         err = instruction_config(p, NULL, instructions, n_instructions);
10577         if (err)
10578                 return err;
10579
10580         /* Thread instruction pointer reset. */
10581         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10582                 struct thread *t = &p->threads[i];
10583
10584                 thread_ip_reset(p, t);
10585         }
10586
10587         return 0;
10588 }
10589
10590 int
10591 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
10592 {
10593         int status;
10594
10595         CHECK(p, EINVAL);
10596         CHECK(p->build_done == 0, EEXIST);
10597
10598         status = port_in_build(p);
10599         if (status)
10600                 goto error;
10601
10602         status = port_out_build(p);
10603         if (status)
10604                 goto error;
10605
10606         status = struct_build(p);
10607         if (status)
10608                 goto error;
10609
10610         status = extern_obj_build(p);
10611         if (status)
10612                 goto error;
10613
10614         status = extern_func_build(p);
10615         if (status)
10616                 goto error;
10617
10618         status = header_build(p);
10619         if (status)
10620                 goto error;
10621
10622         status = metadata_build(p);
10623         if (status)
10624                 goto error;
10625
10626         status = action_build(p);
10627         if (status)
10628                 goto error;
10629
10630         status = table_build(p);
10631         if (status)
10632                 goto error;
10633
10634         status = selector_build(p);
10635         if (status)
10636                 goto error;
10637
10638         status = table_state_build(p);
10639         if (status)
10640                 goto error;
10641
10642         status = regarray_build(p);
10643         if (status)
10644                 goto error;
10645
10646         status = metarray_build(p);
10647         if (status)
10648                 goto error;
10649
10650         p->build_done = 1;
10651         return 0;
10652
10653 error:
10654         metarray_build_free(p);
10655         regarray_build_free(p);
10656         table_state_build_free(p);
10657         selector_build_free(p);
10658         table_build_free(p);
10659         action_build_free(p);
10660         metadata_build_free(p);
10661         header_build_free(p);
10662         extern_func_build_free(p);
10663         extern_obj_build_free(p);
10664         port_out_build_free(p);
10665         port_in_build_free(p);
10666         struct_build_free(p);
10667
10668         return status;
10669 }
10670
10671 void
10672 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
10673 {
10674         uint32_t i;
10675
10676         for (i = 0; i < n_instructions; i++)
10677                 instr_exec(p);
10678 }
10679
10680 void
10681 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
10682 {
10683         uint32_t i;
10684
10685         for (i = 0; i < p->n_ports_out; i++) {
10686                 struct port_out_runtime *port = &p->out[i];
10687
10688                 if (port->flush)
10689                         port->flush(port->obj);
10690         }
10691 }
10692
10693 /*
10694  * Control.
10695  */
10696 int
10697 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
10698                               struct rte_swx_ctl_pipeline_info *pipeline)
10699 {
10700         struct action *action;
10701         struct table *table;
10702         uint32_t n_actions = 0, n_tables = 0;
10703
10704         if (!p || !pipeline)
10705                 return -EINVAL;
10706
10707         TAILQ_FOREACH(action, &p->actions, node)
10708                 n_actions++;
10709
10710         TAILQ_FOREACH(table, &p->tables, node)
10711                 n_tables++;
10712
10713         pipeline->n_ports_in = p->n_ports_in;
10714         pipeline->n_ports_out = p->n_ports_out;
10715         pipeline->n_actions = n_actions;
10716         pipeline->n_tables = n_tables;
10717         pipeline->n_selectors = p->n_selectors;
10718         pipeline->n_regarrays = p->n_regarrays;
10719         pipeline->n_metarrays = p->n_metarrays;
10720
10721         return 0;
10722 }
10723
10724 int
10725 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
10726 {
10727         if (!p || !numa_node)
10728                 return -EINVAL;
10729
10730         *numa_node = p->numa_node;
10731         return 0;
10732 }
10733
10734 int
10735 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
10736                             uint32_t action_id,
10737                             struct rte_swx_ctl_action_info *action)
10738 {
10739         struct action *a = NULL;
10740
10741         if (!p || (action_id >= p->n_actions) || !action)
10742                 return -EINVAL;
10743
10744         a = action_find_by_id(p, action_id);
10745         if (!a)
10746                 return -EINVAL;
10747
10748         strcpy(action->name, a->name);
10749         action->n_args = a->st ? a->st->n_fields : 0;
10750         return 0;
10751 }
10752
10753 int
10754 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10755                                 uint32_t action_id,
10756                                 uint32_t action_arg_id,
10757                                 struct rte_swx_ctl_action_arg_info *action_arg)
10758 {
10759         struct action *a = NULL;
10760         struct field *arg = NULL;
10761
10762         if (!p || (action_id >= p->n_actions) || !action_arg)
10763                 return -EINVAL;
10764
10765         a = action_find_by_id(p, action_id);
10766         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10767                 return -EINVAL;
10768
10769         arg = &a->st->fields[action_arg_id];
10770         strcpy(action_arg->name, arg->name);
10771         action_arg->n_bits = arg->n_bits;
10772         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10773
10774         return 0;
10775 }
10776
10777 int
10778 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10779                            uint32_t table_id,
10780                            struct rte_swx_ctl_table_info *table)
10781 {
10782         struct table *t = NULL;
10783
10784         if (!p || !table)
10785                 return -EINVAL;
10786
10787         t = table_find_by_id(p, table_id);
10788         if (!t)
10789                 return -EINVAL;
10790
10791         strcpy(table->name, t->name);
10792         strcpy(table->args, t->args);
10793         table->n_match_fields = t->n_fields;
10794         table->n_actions = t->n_actions;
10795         table->default_action_is_const = t->default_action_is_const;
10796         table->size = t->size;
10797         return 0;
10798 }
10799
10800 int
10801 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10802         uint32_t table_id,
10803         uint32_t match_field_id,
10804         struct rte_swx_ctl_table_match_field_info *match_field)
10805 {
10806         struct table *t;
10807         struct match_field *f;
10808
10809         if (!p || (table_id >= p->n_tables) || !match_field)
10810                 return -EINVAL;
10811
10812         t = table_find_by_id(p, table_id);
10813         if (!t || (match_field_id >= t->n_fields))
10814                 return -EINVAL;
10815
10816         f = &t->fields[match_field_id];
10817         match_field->match_type = f->match_type;
10818         match_field->is_header = t->header ? 1 : 0;
10819         match_field->n_bits = f->field->n_bits;
10820         match_field->offset = f->field->offset;
10821
10822         return 0;
10823 }
10824
10825 int
10826 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10827         uint32_t table_id,
10828         uint32_t table_action_id,
10829         struct rte_swx_ctl_table_action_info *table_action)
10830 {
10831         struct table *t;
10832
10833         if (!p || (table_id >= p->n_tables) || !table_action)
10834                 return -EINVAL;
10835
10836         t = table_find_by_id(p, table_id);
10837         if (!t || (table_action_id >= t->n_actions))
10838                 return -EINVAL;
10839
10840         table_action->action_id = t->actions[table_action_id]->id;
10841
10842         return 0;
10843 }
10844
10845 int
10846 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10847                           uint32_t table_id,
10848                           struct rte_swx_table_ops *table_ops,
10849                           int *is_stub)
10850 {
10851         struct table *t;
10852
10853         if (!p || (table_id >= p->n_tables))
10854                 return -EINVAL;
10855
10856         t = table_find_by_id(p, table_id);
10857         if (!t)
10858                 return -EINVAL;
10859
10860         if (t->type) {
10861                 if (table_ops)
10862                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10863                 *is_stub = 0;
10864         } else {
10865                 *is_stub = 1;
10866         }
10867
10868         return 0;
10869 }
10870
10871 int
10872 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10873                               uint32_t selector_id,
10874                               struct rte_swx_ctl_selector_info *selector)
10875 {
10876         struct selector *s = NULL;
10877
10878         if (!p || !selector)
10879                 return -EINVAL;
10880
10881         s = selector_find_by_id(p, selector_id);
10882         if (!s)
10883                 return -EINVAL;
10884
10885         strcpy(selector->name, s->name);
10886
10887         selector->n_selector_fields = s->n_selector_fields;
10888         selector->n_groups_max = s->n_groups_max;
10889         selector->n_members_per_group_max = s->n_members_per_group_max;
10890
10891         return 0;
10892 }
10893
10894 int
10895 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10896          uint32_t selector_id,
10897          struct rte_swx_ctl_table_match_field_info *field)
10898 {
10899         struct selector *s;
10900
10901         if (!p || (selector_id >= p->n_selectors) || !field)
10902                 return -EINVAL;
10903
10904         s = selector_find_by_id(p, selector_id);
10905         if (!s)
10906                 return -EINVAL;
10907
10908         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10909         field->is_header = 0;
10910         field->n_bits = s->group_id_field->n_bits;
10911         field->offset = s->group_id_field->offset;
10912
10913         return 0;
10914 }
10915
10916 int
10917 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10918          uint32_t selector_id,
10919          uint32_t selector_field_id,
10920          struct rte_swx_ctl_table_match_field_info *field)
10921 {
10922         struct selector *s;
10923         struct field *f;
10924
10925         if (!p || (selector_id >= p->n_selectors) || !field)
10926                 return -EINVAL;
10927
10928         s = selector_find_by_id(p, selector_id);
10929         if (!s || (selector_field_id >= s->n_selector_fields))
10930                 return -EINVAL;
10931
10932         f = s->selector_fields[selector_field_id];
10933         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10934         field->is_header = s->selector_header ? 1 : 0;
10935         field->n_bits = f->n_bits;
10936         field->offset = f->offset;
10937
10938         return 0;
10939 }
10940
10941 int
10942 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10943          uint32_t selector_id,
10944          struct rte_swx_ctl_table_match_field_info *field)
10945 {
10946         struct selector *s;
10947
10948         if (!p || (selector_id >= p->n_selectors) || !field)
10949                 return -EINVAL;
10950
10951         s = selector_find_by_id(p, selector_id);
10952         if (!s)
10953                 return -EINVAL;
10954
10955         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10956         field->is_header = 0;
10957         field->n_bits = s->member_id_field->n_bits;
10958         field->offset = s->member_id_field->offset;
10959
10960         return 0;
10961 }
10962
10963 int
10964 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10965                                  struct rte_swx_table_state **table_state)
10966 {
10967         if (!p || !table_state || !p->build_done)
10968                 return -EINVAL;
10969
10970         *table_state = p->table_state;
10971         return 0;
10972 }
10973
10974 int
10975 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10976                                  struct rte_swx_table_state *table_state)
10977 {
10978         if (!p || !table_state || !p->build_done)
10979                 return -EINVAL;
10980
10981         p->table_state = table_state;
10982         return 0;
10983 }
10984
10985 int
10986 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10987                                         uint32_t port_id,
10988                                         struct rte_swx_port_in_stats *stats)
10989 {
10990         struct port_in *port;
10991
10992         if (!p || !stats)
10993                 return -EINVAL;
10994
10995         port = port_in_find(p, port_id);
10996         if (!port)
10997                 return -EINVAL;
10998
10999         port->type->ops.stats_read(port->obj, stats);
11000         return 0;
11001 }
11002
11003 int
11004 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
11005                                          uint32_t port_id,
11006                                          struct rte_swx_port_out_stats *stats)
11007 {
11008         struct port_out *port;
11009
11010         if (!p || !stats)
11011                 return -EINVAL;
11012
11013         port = port_out_find(p, port_id);
11014         if (!port)
11015                 return -EINVAL;
11016
11017         port->type->ops.stats_read(port->obj, stats);
11018         return 0;
11019 }
11020
11021 int
11022 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
11023                                       const char *table_name,
11024                                       struct rte_swx_table_stats *stats)
11025 {
11026         struct table *table;
11027         struct table_statistics *table_stats;
11028
11029         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
11030                 return -EINVAL;
11031
11032         table = table_find(p, table_name);
11033         if (!table)
11034                 return -EINVAL;
11035
11036         table_stats = &p->table_stats[table->id];
11037
11038         memcpy(stats->n_pkts_action,
11039                table_stats->n_pkts_action,
11040                p->n_actions * sizeof(uint64_t));
11041
11042         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
11043         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
11044
11045         return 0;
11046 }
11047
11048 int
11049 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
11050         const char *selector_name,
11051         struct rte_swx_pipeline_selector_stats *stats)
11052 {
11053         struct selector *s;
11054
11055         if (!p || !selector_name || !selector_name[0] || !stats)
11056                 return -EINVAL;
11057
11058         s = selector_find(p, selector_name);
11059         if (!s)
11060                 return -EINVAL;
11061
11062         stats->n_pkts = p->selector_stats[s->id].n_pkts;
11063
11064         return 0;
11065 }
11066
11067 int
11068 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
11069                               uint32_t regarray_id,
11070                               struct rte_swx_ctl_regarray_info *regarray)
11071 {
11072         struct regarray *r;
11073
11074         if (!p || !regarray)
11075                 return -EINVAL;
11076
11077         r = regarray_find_by_id(p, regarray_id);
11078         if (!r)
11079                 return -EINVAL;
11080
11081         strcpy(regarray->name, r->name);
11082         regarray->size = r->size;
11083         return 0;
11084 }
11085
11086 int
11087 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
11088                                    const char *regarray_name,
11089                                    uint32_t regarray_index,
11090                                    uint64_t *value)
11091 {
11092         struct regarray *regarray;
11093         struct regarray_runtime *r;
11094
11095         if (!p || !regarray_name || !value)
11096                 return -EINVAL;
11097
11098         regarray = regarray_find(p, regarray_name);
11099         if (!regarray || (regarray_index >= regarray->size))
11100                 return -EINVAL;
11101
11102         r = &p->regarray_runtime[regarray->id];
11103         *value = r->regarray[regarray_index];
11104         return 0;
11105 }
11106
11107 int
11108 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
11109                                    const char *regarray_name,
11110                                    uint32_t regarray_index,
11111                                    uint64_t value)
11112 {
11113         struct regarray *regarray;
11114         struct regarray_runtime *r;
11115
11116         if (!p || !regarray_name)
11117                 return -EINVAL;
11118
11119         regarray = regarray_find(p, regarray_name);
11120         if (!regarray || (regarray_index >= regarray->size))
11121                 return -EINVAL;
11122
11123         r = &p->regarray_runtime[regarray->id];
11124         r->regarray[regarray_index] = value;
11125         return 0;
11126 }
11127
11128 int
11129 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
11130                               uint32_t metarray_id,
11131                               struct rte_swx_ctl_metarray_info *metarray)
11132 {
11133         struct metarray *m;
11134
11135         if (!p || !metarray)
11136                 return -EINVAL;
11137
11138         m = metarray_find_by_id(p, metarray_id);
11139         if (!m)
11140                 return -EINVAL;
11141
11142         strcpy(metarray->name, m->name);
11143         metarray->size = m->size;
11144         return 0;
11145 }
11146
11147 int
11148 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
11149                               const char *name,
11150                               struct rte_meter_trtcm_params *params)
11151 {
11152         struct meter_profile *mp;
11153         int status;
11154
11155         CHECK(p, EINVAL);
11156         CHECK_NAME(name, EINVAL);
11157         CHECK(params, EINVAL);
11158         CHECK(!meter_profile_find(p, name), EEXIST);
11159
11160         /* Node allocation. */
11161         mp = calloc(1, sizeof(struct meter_profile));
11162         CHECK(mp, ENOMEM);
11163
11164         /* Node initialization. */
11165         strcpy(mp->name, name);
11166         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
11167         status = rte_meter_trtcm_profile_config(&mp->profile, params);
11168         if (status) {
11169                 free(mp);
11170                 CHECK(0, EINVAL);
11171         }
11172
11173         /* Node add to tailq. */
11174         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
11175
11176         return 0;
11177 }
11178
11179 int
11180 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
11181                                  const char *name)
11182 {
11183         struct meter_profile *mp;
11184
11185         CHECK(p, EINVAL);
11186         CHECK_NAME(name, EINVAL);
11187
11188         mp = meter_profile_find(p, name);
11189         CHECK(mp, EINVAL);
11190         CHECK(!mp->n_users, EBUSY);
11191
11192         /* Remove node from tailq. */
11193         TAILQ_REMOVE(&p->meter_profiles, mp, node);
11194         free(mp);
11195
11196         return 0;
11197 }
11198
11199 int
11200 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
11201                         const char *metarray_name,
11202                         uint32_t metarray_index)
11203 {
11204         struct meter_profile *mp_old;
11205         struct metarray *metarray;
11206         struct metarray_runtime *metarray_runtime;
11207         struct meter *m;
11208
11209         CHECK(p, EINVAL);
11210         CHECK_NAME(metarray_name, EINVAL);
11211
11212         metarray = metarray_find(p, metarray_name);
11213         CHECK(metarray, EINVAL);
11214         CHECK(metarray_index < metarray->size, EINVAL);
11215
11216         metarray_runtime = &p->metarray_runtime[metarray->id];
11217         m = &metarray_runtime->metarray[metarray_index];
11218         mp_old = m->profile;
11219
11220         meter_init(m);
11221
11222         mp_old->n_users--;
11223
11224         return 0;
11225 }
11226
11227 int
11228 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
11229                       const char *metarray_name,
11230                       uint32_t metarray_index,
11231                       const char *profile_name)
11232 {
11233         struct meter_profile *mp, *mp_old;
11234         struct metarray *metarray;
11235         struct metarray_runtime *metarray_runtime;
11236         struct meter *m;
11237
11238         CHECK(p, EINVAL);
11239         CHECK_NAME(metarray_name, EINVAL);
11240
11241         metarray = metarray_find(p, metarray_name);
11242         CHECK(metarray, EINVAL);
11243         CHECK(metarray_index < metarray->size, EINVAL);
11244
11245         mp = meter_profile_find(p, profile_name);
11246         CHECK(mp, EINVAL);
11247
11248         metarray_runtime = &p->metarray_runtime[metarray->id];
11249         m = &metarray_runtime->metarray[metarray_index];
11250         mp_old = m->profile;
11251
11252         memset(m, 0, sizeof(struct meter));
11253         rte_meter_trtcm_config(&m->m, &mp->profile);
11254         m->profile = mp;
11255         m->color_mask = RTE_COLORS;
11256
11257         mp->n_users++;
11258         mp_old->n_users--;
11259
11260         return 0;
11261 }
11262
11263 int
11264 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
11265                              const char *metarray_name,
11266                              uint32_t metarray_index,
11267                              struct rte_swx_ctl_meter_stats *stats)
11268 {
11269         struct metarray *metarray;
11270         struct metarray_runtime *metarray_runtime;
11271         struct meter *m;
11272
11273         CHECK(p, EINVAL);
11274         CHECK_NAME(metarray_name, EINVAL);
11275
11276         metarray = metarray_find(p, metarray_name);
11277         CHECK(metarray, EINVAL);
11278         CHECK(metarray_index < metarray->size, EINVAL);
11279
11280         CHECK(stats, EINVAL);
11281
11282         metarray_runtime = &p->metarray_runtime[metarray->id];
11283         m = &metarray_runtime->metarray[metarray_index];
11284
11285         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
11286         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
11287
11288         return 0;
11289 }