lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <inttypes.h>
   9 #include <sys/queue.h>
  10 #include <arpa/inet.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_prefetch.h>
  14 #include <rte_byteorder.h>
  15 #include <rte_cycles.h>
  16 #include <rte_meter.h>
  17
  18 #include "rte_swx_pipeline.h"
  19 #include "rte_swx_ctl.h"
  20
  21 #define CHECK(condition, err_code)                                             \
  22 do {                                                                           \
  23         if (!(condition))                                                      \
  24                 return -(err_code);                                            \
  25 } while (0)
  26
  27 #define CHECK_NAME(name, err_code)                                             \
  28         CHECK((name) &&                                                        \
  29               (name)[0] &&                                                     \
  30               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  31               err_code)
  32
  33 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  34         CHECK((instr) &&                                                       \
  35               (instr)[0] &&                                                    \
  36               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  37                RTE_SWX_INSTRUCTION_SIZE),                                      \
  38               err_code)
  39
  40 #ifndef TRACE_LEVEL
  41 #define TRACE_LEVEL 0
  42 #endif
  43
  44 #if TRACE_LEVEL
  45 #define TRACE(...) printf(__VA_ARGS__)
  46 #else
  47 #define TRACE(...)
  48 #endif
  49
  50 /*
  51  * Environment.
  52  */
  53 #define ntoh64(x) rte_be_to_cpu_64(x)
  54 #define hton64(x) rte_cpu_to_be_64(x)
  55
  56 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  57
  58 #include <rte_malloc.h>
  59
  60 static void *
  61 env_malloc(size_t size, size_t alignment, int numa_node)
  62 {
  63         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  64 }
  65
  66 static void
  67 env_free(void *start, size_t size __rte_unused)
  68 {
  69         rte_free(start);
  70 }
  71
  72 #else
  73
  74 #include <numa.h>
  75
  76 static void *
  77 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  78 {
  79         void *start;
  80
  81         if (numa_available() == -1)
  82                 return NULL;
  83
  84         start = numa_alloc_onnode(size, numa_node);
  85         if (!start)
  86                 return NULL;
  87
  88         memset(start, 0, size);
  89         return start;
  90 }
  91
  92 static void
  93 env_free(void *start, size_t size)
  94 {
  95         if (numa_available() == -1)
  96                 return;
  97
  98         numa_free(start, size);
  99 }
 100
 101 #endif
 102
 103 /*
 104  * Struct.
 105  */
 106 struct field {
 107         char name[RTE_SWX_NAME_SIZE];
 108         uint32_t n_bits;
 109         uint32_t offset;
 110 };
 111
 112 struct struct_type {
 113         TAILQ_ENTRY(struct_type) node;
 114         char name[RTE_SWX_NAME_SIZE];
 115         struct field *fields;
 116         uint32_t n_fields;
 117         uint32_t n_bits;
 118 };
 119
 120 TAILQ_HEAD(struct_type_tailq, struct_type);
 121
 122 /*
 123  * Input port.
 124  */
 125 struct port_in_type {
 126         TAILQ_ENTRY(port_in_type) node;
 127         char name[RTE_SWX_NAME_SIZE];
 128         struct rte_swx_port_in_ops ops;
 129 };
 130
 131 TAILQ_HEAD(port_in_type_tailq, port_in_type);
 132
 133 struct port_in {
 134         TAILQ_ENTRY(port_in) node;
 135         struct port_in_type *type;
 136         void *obj;
 137         uint32_t id;
 138 };
 139
 140 TAILQ_HEAD(port_in_tailq, port_in);
 141
 142 struct port_in_runtime {
 143         rte_swx_port_in_pkt_rx_t pkt_rx;
 144         void *obj;
 145 };
 146
 147 /*
 148  * Output port.
 149  */
 150 struct port_out_type {
 151         TAILQ_ENTRY(port_out_type) node;
 152         char name[RTE_SWX_NAME_SIZE];
 153         struct rte_swx_port_out_ops ops;
 154 };
 155
 156 TAILQ_HEAD(port_out_type_tailq, port_out_type);
 157
 158 struct port_out {
 159         TAILQ_ENTRY(port_out) node;
 160         struct port_out_type *type;
 161         void *obj;
 162         uint32_t id;
 163 };
 164
 165 TAILQ_HEAD(port_out_tailq, port_out);
 166
 167 struct port_out_runtime {
 168         rte_swx_port_out_pkt_tx_t pkt_tx;
 169         rte_swx_port_out_flush_t flush;
 170         void *obj;
 171 };
 172
 173 /*
 174  * Extern object.
 175  */
 176 struct extern_type_member_func {
 177         TAILQ_ENTRY(extern_type_member_func) node;
 178         char name[RTE_SWX_NAME_SIZE];
 179         rte_swx_extern_type_member_func_t func;
 180         uint32_t id;
 181 };
 182
 183 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 184
 185 struct extern_type {
 186         TAILQ_ENTRY(extern_type) node;
 187         char name[RTE_SWX_NAME_SIZE];
 188         struct struct_type *mailbox_struct_type;
 189         rte_swx_extern_type_constructor_t constructor;
 190         rte_swx_extern_type_destructor_t destructor;
 191         struct extern_type_member_func_tailq funcs;
 192         uint32_t n_funcs;
 193 };
 194
 195 TAILQ_HEAD(extern_type_tailq, extern_type);
 196
 197 struct extern_obj {
 198         TAILQ_ENTRY(extern_obj) node;
 199         char name[RTE_SWX_NAME_SIZE];
 200         struct extern_type *type;
 201         void *obj;
 202         uint32_t struct_id;
 203         uint32_t id;
 204 };
 205
 206 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 207
 208 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 209 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 210 #endif
 211
 212 struct extern_obj_runtime {
 213         void *obj;
 214         uint8_t *mailbox;
 215         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 216 };
 217
 218 /*
 219  * Extern function.
 220  */
 221 struct extern_func {
 222         TAILQ_ENTRY(extern_func) node;
 223         char name[RTE_SWX_NAME_SIZE];
 224         struct struct_type *mailbox_struct_type;
 225         rte_swx_extern_func_t func;
 226         uint32_t struct_id;
 227         uint32_t id;
 228 };
 229
 230 TAILQ_HEAD(extern_func_tailq, extern_func);
 231
 232 struct extern_func_runtime {
 233         uint8_t *mailbox;
 234         rte_swx_extern_func_t func;
 235 };
 236
 237 /*
 238  * Header.
 239  */
 240 struct header {
 241         TAILQ_ENTRY(header) node;
 242         char name[RTE_SWX_NAME_SIZE];
 243         struct struct_type *st;
 244         uint32_t struct_id;
 245         uint32_t id;
 246 };
 247
 248 TAILQ_HEAD(header_tailq, header);
 249
 250 struct header_runtime {
 251         uint8_t *ptr0;
 252 };
 253
 254 struct header_out_runtime {
 255         uint8_t *ptr0;
 256         uint8_t *ptr;
 257         uint32_t n_bytes;
 258 };
 259
 260 /*
 261  * Instruction.
 262  */
 263
 264 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 265  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 266  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 267  * when transferred to packet meta-data and in NBO when transferred to packet
 268  * headers.
 269  */
 270
 271 /* Notation conventions:
 272  *    -Header field: H = h.header.field (dst/src)
 273  *    -Meta-data field: M = m.field (dst/src)
 274  *    -Extern object mailbox field: E = e.field (dst/src)
 275  *    -Extern function mailbox field: F = f.field (dst/src)
 276  *    -Table action data field: T = t.field (src only)
 277  *    -Immediate value: I = 32-bit unsigned value (src only)
 278  */
 279
 280 enum instruction_type {
 281         /* rx m.port_in */
 282         INSTR_RX,
 283
 284         /* tx port_out
 285          * port_out = MI
 286          */
 287         INSTR_TX,   /* port_out = M */
 288         INSTR_TX_I, /* port_out = I */
 289
 290         /* extract h.header */
 291         INSTR_HDR_EXTRACT,
 292         INSTR_HDR_EXTRACT2,
 293         INSTR_HDR_EXTRACT3,
 294         INSTR_HDR_EXTRACT4,
 295         INSTR_HDR_EXTRACT5,
 296         INSTR_HDR_EXTRACT6,
 297         INSTR_HDR_EXTRACT7,
 298         INSTR_HDR_EXTRACT8,
 299
 300         /* emit h.header */
 301         INSTR_HDR_EMIT,
 302         INSTR_HDR_EMIT_TX,
 303         INSTR_HDR_EMIT2_TX,
 304         INSTR_HDR_EMIT3_TX,
 305         INSTR_HDR_EMIT4_TX,
 306         INSTR_HDR_EMIT5_TX,
 307         INSTR_HDR_EMIT6_TX,
 308         INSTR_HDR_EMIT7_TX,
 309         INSTR_HDR_EMIT8_TX,
 310
 311         /* validate h.header */
 312         INSTR_HDR_VALIDATE,
 313
 314         /* invalidate h.header */
 315         INSTR_HDR_INVALIDATE,
 316
 317         /* mov dst src
 318          * dst = src
 319          * dst = HMEF, src = HMEFTI
 320          */
 321         INSTR_MOV,   /* dst = MEF, src = MEFT */
 322         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 323         INSTR_MOV_I, /* dst = HMEF, src = I */
 324
 325         /* dma h.header t.field
 326          * memcpy(h.header, t.field, sizeof(h.header))
 327          */
 328         INSTR_DMA_HT,
 329         INSTR_DMA_HT2,
 330         INSTR_DMA_HT3,
 331         INSTR_DMA_HT4,
 332         INSTR_DMA_HT5,
 333         INSTR_DMA_HT6,
 334         INSTR_DMA_HT7,
 335         INSTR_DMA_HT8,
 336
 337         /* add dst src
 338          * dst += src
 339          * dst = HMEF, src = HMEFTI
 340          */
 341         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 342         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 343         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 344         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 345         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 346         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 347
 348         /* sub dst src
 349          * dst -= src
 350          * dst = HMEF, src = HMEFTI
 351          */
 352         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 353         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 354         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 355         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 356         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 357         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 358
 359         /* ckadd dst src
 360          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 361          * dst = H, src = {H, h.header}
 362          */
 363         INSTR_ALU_CKADD_FIELD,    /* src = H */
 364         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 365         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 366
 367         /* cksub dst src
 368          * dst = dst '- src
 369          * dst = H, src = H
 370          */
 371         INSTR_ALU_CKSUB_FIELD,
 372
 373         /* and dst src
 374          * dst &= src
 375          * dst = HMEF, src = HMEFTI
 376          */
 377         INSTR_ALU_AND,   /* dst = MEF, src = MEFT */
 378         INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 379         INSTR_ALU_AND_I, /* dst = HMEF, src = I */
 380
 381         /* or dst src
 382          * dst |= src
 383          * dst = HMEF, src = HMEFTI
 384          */
 385         INSTR_ALU_OR,   /* dst = MEF, src = MEFT */
 386         INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 387         INSTR_ALU_OR_I, /* dst = HMEF, src = I */
 388
 389         /* xor dst src
 390          * dst ^= src
 391          * dst = HMEF, src = HMEFTI
 392          */
 393         INSTR_ALU_XOR,   /* dst = MEF, src = MEFT */
 394         INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 395         INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
 396
 397         /* shl dst src
 398          * dst <<= src
 399          * dst = HMEF, src = HMEFTI
 400          */
 401         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 402         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 403         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 404         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 405         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 406         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 407
 408         /* shr dst src
 409          * dst >>= src
 410          * dst = HMEF, src = HMEFTI
 411          */
 412         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 413         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 414         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 415         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 416         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 417         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 418
 419         /* regprefetch REGARRAY index
 420          * prefetch REGARRAY[index]
 421          * index = HMEFTI
 422          */
 423         INSTR_REGPREFETCH_RH, /* index = H */
 424         INSTR_REGPREFETCH_RM, /* index = MEFT */
 425         INSTR_REGPREFETCH_RI, /* index = I */
 426
 427         /* regrd dst REGARRAY index
 428          * dst = REGARRAY[index]
 429          * dst = HMEF, index = HMEFTI
 430          */
 431         INSTR_REGRD_HRH, /* dst = H, index = H */
 432         INSTR_REGRD_HRM, /* dst = H, index = MEFT */
 433         INSTR_REGRD_HRI, /* dst = H, index = I */
 434         INSTR_REGRD_MRH, /* dst = MEF, index = H */
 435         INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
 436         INSTR_REGRD_MRI, /* dst = MEF, index = I */
 437
 438         /* regwr REGARRAY index src
 439          * REGARRAY[index] = src
 440          * index = HMEFTI, src = HMEFTI
 441          */
 442         INSTR_REGWR_RHH, /* index = H, src = H */
 443         INSTR_REGWR_RHM, /* index = H, src = MEFT */
 444         INSTR_REGWR_RHI, /* index = H, src = I */
 445         INSTR_REGWR_RMH, /* index = MEFT, src = H */
 446         INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
 447         INSTR_REGWR_RMI, /* index = MEFT, src = I */
 448         INSTR_REGWR_RIH, /* index = I, src = H */
 449         INSTR_REGWR_RIM, /* index = I, src = MEFT */
 450         INSTR_REGWR_RII, /* index = I, src = I */
 451
 452         /* regadd REGARRAY index src
 453          * REGARRAY[index] += src
 454          * index = HMEFTI, src = HMEFTI
 455          */
 456         INSTR_REGADD_RHH, /* index = H, src = H */
 457         INSTR_REGADD_RHM, /* index = H, src = MEFT */
 458         INSTR_REGADD_RHI, /* index = H, src = I */
 459         INSTR_REGADD_RMH, /* index = MEFT, src = H */
 460         INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
 461         INSTR_REGADD_RMI, /* index = MEFT, src = I */
 462         INSTR_REGADD_RIH, /* index = I, src = H */
 463         INSTR_REGADD_RIM, /* index = I, src = MEFT */
 464         INSTR_REGADD_RII, /* index = I, src = I */
 465
 466         /* metprefetch METARRAY index
 467          * prefetch METARRAY[index]
 468          * index = HMEFTI
 469          */
 470         INSTR_METPREFETCH_H, /* index = H */
 471         INSTR_METPREFETCH_M, /* index = MEFT */
 472         INSTR_METPREFETCH_I, /* index = I */
 473
 474         /* meter METARRAY index length color_in color_out
 475          * color_out = meter(METARRAY[index], length, color_in)
 476          * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
 477          */
 478         INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
 479         INSTR_METER_HHI, /* index = H, length = H, color_in = I */
 480         INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
 481         INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
 482         INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
 483         INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
 484         INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
 485         INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
 486         INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
 487         INSTR_METER_IHI, /* index = I, length = H, color_in = I */
 488         INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
 489         INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
 490
 491         /* table TABLE */
 492         INSTR_TABLE,
 493
 494         /* extern e.obj.func */
 495         INSTR_EXTERN_OBJ,
 496
 497         /* extern f.func */
 498         INSTR_EXTERN_FUNC,
 499
 500         /* jmp LABEL
 501          * Unconditional jump
 502          */
 503         INSTR_JMP,
 504
 505         /* jmpv LABEL h.header
 506          * Jump if header is valid
 507          */
 508         INSTR_JMP_VALID,
 509
 510         /* jmpnv LABEL h.header
 511          * Jump if header is invalid
 512          */
 513         INSTR_JMP_INVALID,
 514
 515         /* jmph LABEL
 516          * Jump if table lookup hit
 517          */
 518         INSTR_JMP_HIT,
 519
 520         /* jmpnh LABEL
 521          * Jump if table lookup miss
 522          */
 523         INSTR_JMP_MISS,
 524
 525         /* jmpa LABEL ACTION
 526          * Jump if action run
 527          */
 528         INSTR_JMP_ACTION_HIT,
 529
 530         /* jmpna LABEL ACTION
 531          * Jump if action not run
 532          */
 533         INSTR_JMP_ACTION_MISS,
 534
 535         /* jmpeq LABEL a b
 536          * Jump is a is equal to b
 537          * a = HMEFT, b = HMEFTI
 538          */
 539         INSTR_JMP_EQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
 540         INSTR_JMP_EQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
 541         INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 542
 543         /* jmpneq LABEL a b
 544          * Jump is a is not equal to b
 545          * a = HMEFT, b = HMEFTI
 546          */
 547         INSTR_JMP_NEQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
 548         INSTR_JMP_NEQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
 549         INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 550
 551         /* jmplt LABEL a b
 552          * Jump if a is less than b
 553          * a = HMEFT, b = HMEFTI
 554          */
 555         INSTR_JMP_LT,    /* a = MEF, b = MEF */
 556         INSTR_JMP_LT_MH, /* a = MEF, b = H */
 557         INSTR_JMP_LT_HM, /* a = H, b = MEF */
 558         INSTR_JMP_LT_HH, /* a = H, b = H */
 559         INSTR_JMP_LT_MI, /* a = MEF, b = I */
 560         INSTR_JMP_LT_HI, /* a = H, b = I */
 561
 562         /* jmpgt LABEL a b
 563          * Jump if a is greater than b
 564          * a = HMEFT, b = HMEFTI
 565          */
 566         INSTR_JMP_GT,    /* a = MEF, b = MEF */
 567         INSTR_JMP_GT_MH, /* a = MEF, b = H */
 568         INSTR_JMP_GT_HM, /* a = H, b = MEF */
 569         INSTR_JMP_GT_HH, /* a = H, b = H */
 570         INSTR_JMP_GT_MI, /* a = MEF, b = I */
 571         INSTR_JMP_GT_HI, /* a = H, b = I */
 572
 573         /* return
 574          * Return from action
 575          */
 576         INSTR_RETURN,
 577 };
 578
 579 struct instr_operand {
 580         uint8_t struct_id;
 581         uint8_t n_bits;
 582         uint8_t offset;
 583         uint8_t pad;
 584 };
 585
 586 struct instr_io {
 587         struct {
 588                 union {
 589                         struct {
 590                                 uint8_t offset;
 591                                 uint8_t n_bits;
 592                                 uint8_t pad[2];
 593                         };
 594
 595                         uint32_t val;
 596                 };
 597         } io;
 598
 599         struct {
 600                 uint8_t header_id[8];
 601                 uint8_t struct_id[8];
 602                 uint8_t n_bytes[8];
 603         } hdr;
 604 };
 605
 606 struct instr_hdr_validity {
 607         uint8_t header_id;
 608 };
 609
 610 struct instr_table {
 611         uint8_t table_id;
 612 };
 613
 614 struct instr_extern_obj {
 615         uint8_t ext_obj_id;
 616         uint8_t func_id;
 617 };
 618
 619 struct instr_extern_func {
 620         uint8_t ext_func_id;
 621 };
 622
 623 struct instr_dst_src {
 624         struct instr_operand dst;
 625         union {
 626                 struct instr_operand src;
 627                 uint64_t src_val;
 628         };
 629 };
 630
 631 struct instr_regarray {
 632         uint8_t regarray_id;
 633         uint8_t pad[3];
 634
 635         union {
 636                 struct instr_operand idx;
 637                 uint32_t idx_val;
 638         };
 639
 640         union {
 641                 struct instr_operand dstsrc;
 642                 uint64_t dstsrc_val;
 643         };
 644 };
 645
 646 struct instr_meter {
 647         uint8_t metarray_id;
 648         uint8_t pad[3];
 649
 650         union {
 651                 struct instr_operand idx;
 652                 uint32_t idx_val;
 653         };
 654
 655         struct instr_operand length;
 656
 657         union {
 658                 struct instr_operand color_in;
 659                 uint32_t color_in_val;
 660         };
 661
 662         struct instr_operand color_out;
 663 };
 664
 665 struct instr_dma {
 666         struct {
 667                 uint8_t header_id[8];
 668                 uint8_t struct_id[8];
 669         } dst;
 670
 671         struct {
 672                 uint8_t offset[8];
 673         } src;
 674
 675         uint16_t n_bytes[8];
 676 };
 677
 678 struct instr_jmp {
 679         struct instruction *ip;
 680
 681         union {
 682                 struct instr_operand a;
 683                 uint8_t header_id;
 684                 uint8_t action_id;
 685         };
 686
 687         union {
 688                 struct instr_operand b;
 689                 uint64_t b_val;
 690         };
 691 };
 692
 693 struct instruction {
 694         enum instruction_type type;
 695         union {
 696                 struct instr_io io;
 697                 struct instr_hdr_validity valid;
 698                 struct instr_dst_src mov;
 699                 struct instr_regarray regarray;
 700                 struct instr_meter meter;
 701                 struct instr_dma dma;
 702                 struct instr_dst_src alu;
 703                 struct instr_table table;
 704                 struct instr_extern_obj ext_obj;
 705                 struct instr_extern_func ext_func;
 706                 struct instr_jmp jmp;
 707         };
 708 };
 709
 710 struct instruction_data {
 711         char label[RTE_SWX_NAME_SIZE];
 712         char jmp_label[RTE_SWX_NAME_SIZE];
 713         uint32_t n_users; /* user = jmp instruction to this instruction. */
 714         int invalid;
 715 };
 716
 717 /*
 718  * Action.
 719  */
 720 struct action {
 721         TAILQ_ENTRY(action) node;
 722         char name[RTE_SWX_NAME_SIZE];
 723         struct struct_type *st;
 724         struct instruction *instructions;
 725         uint32_t n_instructions;
 726         uint32_t id;
 727 };
 728
 729 TAILQ_HEAD(action_tailq, action);
 730
 731 /*
 732  * Table.
 733  */
 734 struct table_type {
 735         TAILQ_ENTRY(table_type) node;
 736         char name[RTE_SWX_NAME_SIZE];
 737         enum rte_swx_table_match_type match_type;
 738         struct rte_swx_table_ops ops;
 739 };
 740
 741 TAILQ_HEAD(table_type_tailq, table_type);
 742
 743 struct match_field {
 744         enum rte_swx_table_match_type match_type;
 745         struct field *field;
 746 };
 747
 748 struct table {
 749         TAILQ_ENTRY(table) node;
 750         char name[RTE_SWX_NAME_SIZE];
 751         char args[RTE_SWX_NAME_SIZE];
 752         struct table_type *type; /* NULL when n_fields == 0. */
 753
 754         /* Match. */
 755         struct match_field *fields;
 756         uint32_t n_fields;
 757         struct header *header; /* Only valid when n_fields > 0. */
 758
 759         /* Action. */
 760         struct action **actions;
 761         struct action *default_action;
 762         uint8_t *default_action_data;
 763         uint32_t n_actions;
 764         int default_action_is_const;
 765         uint32_t action_data_size_max;
 766
 767         uint32_t size;
 768         uint32_t id;
 769 };
 770
 771 TAILQ_HEAD(table_tailq, table);
 772
 773 struct table_runtime {
 774         rte_swx_table_lookup_t func;
 775         void *mailbox;
 776         uint8_t **key;
 777 };
 778
 779 struct table_statistics {
 780         uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
 781         uint64_t *n_pkts_action;
 782 };
 783
 784 /*
 785  * Register array.
 786  */
 787 struct regarray {
 788         TAILQ_ENTRY(regarray) node;
 789         char name[RTE_SWX_NAME_SIZE];
 790         uint64_t init_val;
 791         uint32_t size;
 792         uint32_t id;
 793 };
 794
 795 TAILQ_HEAD(regarray_tailq, regarray);
 796
 797 struct regarray_runtime {
 798         uint64_t *regarray;
 799         uint32_t size_mask;
 800 };
 801
 802 /*
 803  * Meter array.
 804  */
 805 struct meter_profile {
 806         TAILQ_ENTRY(meter_profile) node;
 807         char name[RTE_SWX_NAME_SIZE];
 808         struct rte_meter_trtcm_params params;
 809         struct rte_meter_trtcm_profile profile;
 810         uint32_t n_users;
 811 };
 812
 813 TAILQ_HEAD(meter_profile_tailq, meter_profile);
 814
 815 struct metarray {
 816         TAILQ_ENTRY(metarray) node;
 817         char name[RTE_SWX_NAME_SIZE];
 818         uint32_t size;
 819         uint32_t id;
 820 };
 821
 822 TAILQ_HEAD(metarray_tailq, metarray);
 823
 824 struct meter {
 825         struct rte_meter_trtcm m;
 826         struct meter_profile *profile;
 827         enum rte_color color_mask;
 828         uint8_t pad[20];
 829
 830         uint64_t n_pkts[RTE_COLORS];
 831         uint64_t n_bytes[RTE_COLORS];
 832 };
 833
 834 struct metarray_runtime {
 835         struct meter *metarray;
 836         uint32_t size_mask;
 837 };
 838
 839 /*
 840  * Pipeline.
 841  */
 842 struct thread {
 843         /* Packet. */
 844         struct rte_swx_pkt pkt;
 845         uint8_t *ptr;
 846
 847         /* Structures. */
 848         uint8_t **structs;
 849
 850         /* Packet headers. */
 851         struct header_runtime *headers; /* Extracted or generated headers. */
 852         struct header_out_runtime *headers_out; /* Emitted headers. */
 853         uint8_t *header_storage;
 854         uint8_t *header_out_storage;
 855         uint64_t valid_headers;
 856         uint32_t n_headers_out;
 857
 858         /* Packet meta-data. */
 859         uint8_t *metadata;
 860
 861         /* Tables. */
 862         struct table_runtime *tables;
 863         struct rte_swx_table_state *table_state;
 864         uint64_t action_id;
 865         int hit; /* 0 = Miss, 1 = Hit. */
 866
 867         /* Extern objects and functions. */
 868         struct extern_obj_runtime *extern_objs;
 869         struct extern_func_runtime *extern_funcs;
 870
 871         /* Instructions. */
 872         struct instruction *ip;
 873         struct instruction *ret;
 874 };
 875
 876 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 877 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 878 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 879
 880 #define HEADER_VALID(thread, header_id) \
 881         MASK64_BIT_GET((thread)->valid_headers, header_id)
 882
 883 #define ALU(thread, ip, operator)  \
 884 {                                                                              \
 885         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 886         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 887         uint64_t dst64 = *dst64_ptr;                                           \
 888         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 889         uint64_t dst = dst64 & dst64_mask;                                     \
 890                                                                                \
 891         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 892         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 893         uint64_t src64 = *src64_ptr;                                           \
 894         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 895         uint64_t src = src64 & src64_mask;                                     \
 896                                                                                \
 897         uint64_t result = dst operator src;                                    \
 898                                                                                \
 899         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 900 }
 901
 902 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 903
 904 #define ALU_S(thread, ip, operator)  \
 905 {                                                                              \
 906         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 907         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 908         uint64_t dst64 = *dst64_ptr;                                           \
 909         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 910         uint64_t dst = dst64 & dst64_mask;                                     \
 911                                                                                \
 912         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 913         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 914         uint64_t src64 = *src64_ptr;                                           \
 915         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 916                                                                                \
 917         uint64_t result = dst operator src;                                    \
 918                                                                                \
 919         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 920 }
 921
 922 #define ALU_MH ALU_S
 923
 924 #define ALU_HM(thread, ip, operator)  \
 925 {                                                                              \
 926         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 927         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 928         uint64_t dst64 = *dst64_ptr;                                           \
 929         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 930         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 931                                                                                \
 932         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 933         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 934         uint64_t src64 = *src64_ptr;                                           \
 935         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 936         uint64_t src = src64 & src64_mask;                                     \
 937                                                                                \
 938         uint64_t result = dst operator src;                                    \
 939         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 940                                                                                \
 941         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 942 }
 943
 944 #define ALU_HH(thread, ip, operator)  \
 945 {                                                                              \
 946         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 947         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 948         uint64_t dst64 = *dst64_ptr;                                           \
 949         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 950         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 951                                                                                \
 952         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 953         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 954         uint64_t src64 = *src64_ptr;                                           \
 955         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 956                                                                                \
 957         uint64_t result = dst operator src;                                    \
 958         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 959                                                                                \
 960         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 961 }
 962
 963 #else
 964
 965 #define ALU_S ALU
 966 #define ALU_MH ALU
 967 #define ALU_HM ALU
 968 #define ALU_HH ALU
 969
 970 #endif
 971
 972 #define ALU_I(thread, ip, operator)  \
 973 {                                                                              \
 974         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 975         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 976         uint64_t dst64 = *dst64_ptr;                                           \
 977         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 978         uint64_t dst = dst64 & dst64_mask;                                     \
 979                                                                                \
 980         uint64_t src = (ip)->alu.src_val;                                      \
 981                                                                                \
 982         uint64_t result = dst operator src;                                    \
 983                                                                                \
 984         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 985 }
 986
 987 #define ALU_MI ALU_I
 988
 989 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 990
 991 #define ALU_HI(thread, ip, operator)  \
 992 {                                                                              \
 993         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 994         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 995         uint64_t dst64 = *dst64_ptr;                                           \
 996         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 997         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 998                                                                                \
 999         uint64_t src = (ip)->alu.src_val;                                      \
1000                                                                                \
1001         uint64_t result = dst operator src;                                    \
1002         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1003                                                                                \
1004         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1005 }
1006
1007 #else
1008
1009 #define ALU_HI ALU_I
1010
1011 #endif
1012
1013 #define MOV(thread, ip)  \
1014 {                                                                              \
1015         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1016         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1017         uint64_t dst64 = *dst64_ptr;                                           \
1018         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1019                                                                                \
1020         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1021         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1022         uint64_t src64 = *src64_ptr;                                           \
1023         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1024         uint64_t src = src64 & src64_mask;                                     \
1025                                                                                \
1026         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1027 }
1028
1029 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1030
1031 #define MOV_S(thread, ip)  \
1032 {                                                                              \
1033         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1034         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1035         uint64_t dst64 = *dst64_ptr;                                           \
1036         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1037                                                                                \
1038         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1039         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1040         uint64_t src64 = *src64_ptr;                                           \
1041         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
1042                                                                                \
1043         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1044 }
1045
1046 #else
1047
1048 #define MOV_S MOV
1049
1050 #endif
1051
1052 #define MOV_I(thread, ip)  \
1053 {                                                                              \
1054         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1055         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1056         uint64_t dst64 = *dst64_ptr;                                           \
1057         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1058                                                                                \
1059         uint64_t src = (ip)->mov.src_val;                                      \
1060                                                                                \
1061         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1062 }
1063
1064 #define JMP_CMP(thread, ip, operator)  \
1065 {                                                                              \
1066         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1067         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1068         uint64_t a64 = *a64_ptr;                                               \
1069         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1070         uint64_t a = a64 & a64_mask;                                           \
1071                                                                                \
1072         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1073         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1074         uint64_t b64 = *b64_ptr;                                               \
1075         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1076         uint64_t b = b64 & b64_mask;                                           \
1077                                                                                \
1078         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1079 }
1080
1081 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1082
1083 #define JMP_CMP_S(thread, ip, operator)  \
1084 {                                                                              \
1085         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1086         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1087         uint64_t a64 = *a64_ptr;                                               \
1088         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1089         uint64_t a = a64 & a64_mask;                                           \
1090                                                                                \
1091         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1092         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1093         uint64_t b64 = *b64_ptr;                                               \
1094         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1095                                                                                \
1096         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1097 }
1098
1099 #define JMP_CMP_MH JMP_CMP_S
1100
1101 #define JMP_CMP_HM(thread, ip, operator)  \
1102 {                                                                              \
1103         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1104         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1105         uint64_t a64 = *a64_ptr;                                               \
1106         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1107                                                                                \
1108         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1109         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1110         uint64_t b64 = *b64_ptr;                                               \
1111         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1112         uint64_t b = b64 & b64_mask;                                           \
1113                                                                                \
1114         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1115 }
1116
1117 #define JMP_CMP_HH(thread, ip, operator)  \
1118 {                                                                              \
1119         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1120         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1121         uint64_t a64 = *a64_ptr;                                               \
1122         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1123                                                                                \
1124         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1125         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1126         uint64_t b64 = *b64_ptr;                                               \
1127         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1128                                                                                \
1129         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1130 }
1131
1132 #else
1133
1134 #define JMP_CMP_S JMP_CMP
1135 #define JMP_CMP_MH JMP_CMP
1136 #define JMP_CMP_HM JMP_CMP
1137 #define JMP_CMP_HH JMP_CMP
1138
1139 #endif
1140
1141 #define JMP_CMP_I(thread, ip, operator)  \
1142 {                                                                              \
1143         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1144         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1145         uint64_t a64 = *a64_ptr;                                               \
1146         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1147         uint64_t a = a64 & a64_mask;                                           \
1148                                                                                \
1149         uint64_t b = (ip)->jmp.b_val;                                          \
1150                                                                                \
1151         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1152 }
1153
1154 #define JMP_CMP_MI JMP_CMP_I
1155
1156 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1157
1158 #define JMP_CMP_HI(thread, ip, operator)  \
1159 {                                                                              \
1160         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1161         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1162         uint64_t a64 = *a64_ptr;                                               \
1163         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1164                                                                                \
1165         uint64_t b = (ip)->jmp.b_val;                                          \
1166                                                                                \
1167         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1168 }
1169
1170 #else
1171
1172 #define JMP_CMP_HI JMP_CMP_I
1173
1174 #endif
1175
1176 #define METADATA_READ(thread, offset, n_bits)                                  \
1177 ({                                                                             \
1178         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1179         uint64_t m64 = *m64_ptr;                                               \
1180         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1181         (m64 & m64_mask);                                                      \
1182 })
1183
1184 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
1185 {                                                                              \
1186         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1187         uint64_t m64 = *m64_ptr;                                               \
1188         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1189                                                                                \
1190         uint64_t m_new = value;                                                \
1191                                                                                \
1192         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
1193 }
1194
1195 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1196 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1197 #endif
1198
1199 struct rte_swx_pipeline {
1200         struct struct_type_tailq struct_types;
1201         struct port_in_type_tailq port_in_types;
1202         struct port_in_tailq ports_in;
1203         struct port_out_type_tailq port_out_types;
1204         struct port_out_tailq ports_out;
1205         struct extern_type_tailq extern_types;
1206         struct extern_obj_tailq extern_objs;
1207         struct extern_func_tailq extern_funcs;
1208         struct header_tailq headers;
1209         struct struct_type *metadata_st;
1210         uint32_t metadata_struct_id;
1211         struct action_tailq actions;
1212         struct table_type_tailq table_types;
1213         struct table_tailq tables;
1214         struct regarray_tailq regarrays;
1215         struct meter_profile_tailq meter_profiles;
1216         struct metarray_tailq metarrays;
1217
1218         struct port_in_runtime *in;
1219         struct port_out_runtime *out;
1220         struct instruction **action_instructions;
1221         struct rte_swx_table_state *table_state;
1222         struct table_statistics *table_stats;
1223         struct regarray_runtime *regarray_runtime;
1224         struct metarray_runtime *metarray_runtime;
1225         struct instruction *instructions;
1226         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1227
1228         uint32_t n_structs;
1229         uint32_t n_ports_in;
1230         uint32_t n_ports_out;
1231         uint32_t n_extern_objs;
1232         uint32_t n_extern_funcs;
1233         uint32_t n_actions;
1234         uint32_t n_tables;
1235         uint32_t n_regarrays;
1236         uint32_t n_metarrays;
1237         uint32_t n_headers;
1238         uint32_t thread_id;
1239         uint32_t port_id;
1240         uint32_t n_instructions;
1241         int build_done;
1242         int numa_node;
1243 };
1244
1245 /*
1246  * Struct.
1247  */
1248 static struct struct_type *
1249 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1250 {
1251         struct struct_type *elem;
1252
1253         TAILQ_FOREACH(elem, &p->struct_types, node)
1254                 if (strcmp(elem->name, name) == 0)
1255                         return elem;
1256
1257         return NULL;
1258 }
1259
1260 static struct field *
1261 struct_type_field_find(struct struct_type *st, const char *name)
1262 {
1263         uint32_t i;
1264
1265         for (i = 0; i < st->n_fields; i++) {
1266                 struct field *f = &st->fields[i];
1267
1268                 if (strcmp(f->name, name) == 0)
1269                         return f;
1270         }
1271
1272         return NULL;
1273 }
1274
1275 int
1276 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1277                                       const char *name,
1278                                       struct rte_swx_field_params *fields,
1279                                       uint32_t n_fields)
1280 {
1281         struct struct_type *st;
1282         uint32_t i;
1283
1284         CHECK(p, EINVAL);
1285         CHECK_NAME(name, EINVAL);
1286         CHECK(fields, EINVAL);
1287         CHECK(n_fields, EINVAL);
1288
1289         for (i = 0; i < n_fields; i++) {
1290                 struct rte_swx_field_params *f = &fields[i];
1291                 uint32_t j;
1292
1293                 CHECK_NAME(f->name, EINVAL);
1294                 CHECK(f->n_bits, EINVAL);
1295                 CHECK(f->n_bits <= 64, EINVAL);
1296                 CHECK((f->n_bits & 7) == 0, EINVAL);
1297
1298                 for (j = 0; j < i; j++) {
1299                         struct rte_swx_field_params *f_prev = &fields[j];
1300
1301                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
1302                 }
1303         }
1304
1305         CHECK(!struct_type_find(p, name), EEXIST);
1306
1307         /* Node allocation. */
1308         st = calloc(1, sizeof(struct struct_type));
1309         CHECK(st, ENOMEM);
1310
1311         st->fields = calloc(n_fields, sizeof(struct field));
1312         if (!st->fields) {
1313                 free(st);
1314                 CHECK(0, ENOMEM);
1315         }
1316
1317         /* Node initialization. */
1318         strcpy(st->name, name);
1319         for (i = 0; i < n_fields; i++) {
1320                 struct field *dst = &st->fields[i];
1321                 struct rte_swx_field_params *src = &fields[i];
1322
1323                 strcpy(dst->name, src->name);
1324                 dst->n_bits = src->n_bits;
1325                 dst->offset = st->n_bits;
1326
1327                 st->n_bits += src->n_bits;
1328         }
1329         st->n_fields = n_fields;
1330
1331         /* Node add to tailq. */
1332         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1333
1334         return 0;
1335 }
1336
1337 static int
1338 struct_build(struct rte_swx_pipeline *p)
1339 {
1340         uint32_t i;
1341
1342         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1343                 struct thread *t = &p->threads[i];
1344
1345                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1346                 CHECK(t->structs, ENOMEM);
1347         }
1348
1349         return 0;
1350 }
1351
1352 static void
1353 struct_build_free(struct rte_swx_pipeline *p)
1354 {
1355         uint32_t i;
1356
1357         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1358                 struct thread *t = &p->threads[i];
1359
1360                 free(t->structs);
1361                 t->structs = NULL;
1362         }
1363 }
1364
1365 static void
1366 struct_free(struct rte_swx_pipeline *p)
1367 {
1368         struct_build_free(p);
1369
1370         /* Struct types. */
1371         for ( ; ; ) {
1372                 struct struct_type *elem;
1373
1374                 elem = TAILQ_FIRST(&p->struct_types);
1375                 if (!elem)
1376                         break;
1377
1378                 TAILQ_REMOVE(&p->struct_types, elem, node);
1379                 free(elem->fields);
1380                 free(elem);
1381         }
1382 }
1383
1384 /*
1385  * Input port.
1386  */
1387 static struct port_in_type *
1388 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1389 {
1390         struct port_in_type *elem;
1391
1392         if (!name)
1393                 return NULL;
1394
1395         TAILQ_FOREACH(elem, &p->port_in_types, node)
1396                 if (strcmp(elem->name, name) == 0)
1397                         return elem;
1398
1399         return NULL;
1400 }
1401
1402 int
1403 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1404                                        const char *name,
1405                                        struct rte_swx_port_in_ops *ops)
1406 {
1407         struct port_in_type *elem;
1408
1409         CHECK(p, EINVAL);
1410         CHECK_NAME(name, EINVAL);
1411         CHECK(ops, EINVAL);
1412         CHECK(ops->create, EINVAL);
1413         CHECK(ops->free, EINVAL);
1414         CHECK(ops->pkt_rx, EINVAL);
1415         CHECK(ops->stats_read, EINVAL);
1416
1417         CHECK(!port_in_type_find(p, name), EEXIST);
1418
1419         /* Node allocation. */
1420         elem = calloc(1, sizeof(struct port_in_type));
1421         CHECK(elem, ENOMEM);
1422
1423         /* Node initialization. */
1424         strcpy(elem->name, name);
1425         memcpy(&elem->ops, ops, sizeof(*ops));
1426
1427         /* Node add to tailq. */
1428         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1429
1430         return 0;
1431 }
1432
1433 static struct port_in *
1434 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1435 {
1436         struct port_in *port;
1437
1438         TAILQ_FOREACH(port, &p->ports_in, node)
1439                 if (port->id == port_id)
1440                         return port;
1441
1442         return NULL;
1443 }
1444
1445 int
1446 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1447                                 uint32_t port_id,
1448                                 const char *port_type_name,
1449                                 void *args)
1450 {
1451         struct port_in_type *type = NULL;
1452         struct port_in *port = NULL;
1453         void *obj = NULL;
1454
1455         CHECK(p, EINVAL);
1456
1457         CHECK(!port_in_find(p, port_id), EINVAL);
1458
1459         CHECK_NAME(port_type_name, EINVAL);
1460         type = port_in_type_find(p, port_type_name);
1461         CHECK(type, EINVAL);
1462
1463         obj = type->ops.create(args);
1464         CHECK(obj, ENODEV);
1465
1466         /* Node allocation. */
1467         port = calloc(1, sizeof(struct port_in));
1468         CHECK(port, ENOMEM);
1469
1470         /* Node initialization. */
1471         port->type = type;
1472         port->obj = obj;
1473         port->id = port_id;
1474
1475         /* Node add to tailq. */
1476         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1477         if (p->n_ports_in < port_id + 1)
1478                 p->n_ports_in = port_id + 1;
1479
1480         return 0;
1481 }
1482
1483 static int
1484 port_in_build(struct rte_swx_pipeline *p)
1485 {
1486         struct port_in *port;
1487         uint32_t i;
1488
1489         CHECK(p->n_ports_in, EINVAL);
1490         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1491
1492         for (i = 0; i < p->n_ports_in; i++)
1493                 CHECK(port_in_find(p, i), EINVAL);
1494
1495         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1496         CHECK(p->in, ENOMEM);
1497
1498         TAILQ_FOREACH(port, &p->ports_in, node) {
1499                 struct port_in_runtime *in = &p->in[port->id];
1500
1501                 in->pkt_rx = port->type->ops.pkt_rx;
1502                 in->obj = port->obj;
1503         }
1504
1505         return 0;
1506 }
1507
1508 static void
1509 port_in_build_free(struct rte_swx_pipeline *p)
1510 {
1511         free(p->in);
1512         p->in = NULL;
1513 }
1514
1515 static void
1516 port_in_free(struct rte_swx_pipeline *p)
1517 {
1518         port_in_build_free(p);
1519
1520         /* Input ports. */
1521         for ( ; ; ) {
1522                 struct port_in *port;
1523
1524                 port = TAILQ_FIRST(&p->ports_in);
1525                 if (!port)
1526                         break;
1527
1528                 TAILQ_REMOVE(&p->ports_in, port, node);
1529                 port->type->ops.free(port->obj);
1530                 free(port);
1531         }
1532
1533         /* Input port types. */
1534         for ( ; ; ) {
1535                 struct port_in_type *elem;
1536
1537                 elem = TAILQ_FIRST(&p->port_in_types);
1538                 if (!elem)
1539                         break;
1540
1541                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1542                 free(elem);
1543         }
1544 }
1545
1546 /*
1547  * Output port.
1548  */
1549 static struct port_out_type *
1550 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1551 {
1552         struct port_out_type *elem;
1553
1554         if (!name)
1555                 return NULL;
1556
1557         TAILQ_FOREACH(elem, &p->port_out_types, node)
1558                 if (!strcmp(elem->name, name))
1559                         return elem;
1560
1561         return NULL;
1562 }
1563
1564 int
1565 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1566                                         const char *name,
1567                                         struct rte_swx_port_out_ops *ops)
1568 {
1569         struct port_out_type *elem;
1570
1571         CHECK(p, EINVAL);
1572         CHECK_NAME(name, EINVAL);
1573         CHECK(ops, EINVAL);
1574         CHECK(ops->create, EINVAL);
1575         CHECK(ops->free, EINVAL);
1576         CHECK(ops->pkt_tx, EINVAL);
1577         CHECK(ops->stats_read, EINVAL);
1578
1579         CHECK(!port_out_type_find(p, name), EEXIST);
1580
1581         /* Node allocation. */
1582         elem = calloc(1, sizeof(struct port_out_type));
1583         CHECK(elem, ENOMEM);
1584
1585         /* Node initialization. */
1586         strcpy(elem->name, name);
1587         memcpy(&elem->ops, ops, sizeof(*ops));
1588
1589         /* Node add to tailq. */
1590         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1591
1592         return 0;
1593 }
1594
1595 static struct port_out *
1596 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1597 {
1598         struct port_out *port;
1599
1600         TAILQ_FOREACH(port, &p->ports_out, node)
1601                 if (port->id == port_id)
1602                         return port;
1603
1604         return NULL;
1605 }
1606
1607 int
1608 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1609                                  uint32_t port_id,
1610                                  const char *port_type_name,
1611                                  void *args)
1612 {
1613         struct port_out_type *type = NULL;
1614         struct port_out *port = NULL;
1615         void *obj = NULL;
1616
1617         CHECK(p, EINVAL);
1618
1619         CHECK(!port_out_find(p, port_id), EINVAL);
1620
1621         CHECK_NAME(port_type_name, EINVAL);
1622         type = port_out_type_find(p, port_type_name);
1623         CHECK(type, EINVAL);
1624
1625         obj = type->ops.create(args);
1626         CHECK(obj, ENODEV);
1627
1628         /* Node allocation. */
1629         port = calloc(1, sizeof(struct port_out));
1630         CHECK(port, ENOMEM);
1631
1632         /* Node initialization. */
1633         port->type = type;
1634         port->obj = obj;
1635         port->id = port_id;
1636
1637         /* Node add to tailq. */
1638         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1639         if (p->n_ports_out < port_id + 1)
1640                 p->n_ports_out = port_id + 1;
1641
1642         return 0;
1643 }
1644
1645 static int
1646 port_out_build(struct rte_swx_pipeline *p)
1647 {
1648         struct port_out *port;
1649         uint32_t i;
1650
1651         CHECK(p->n_ports_out, EINVAL);
1652
1653         for (i = 0; i < p->n_ports_out; i++)
1654                 CHECK(port_out_find(p, i), EINVAL);
1655
1656         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1657         CHECK(p->out, ENOMEM);
1658
1659         TAILQ_FOREACH(port, &p->ports_out, node) {
1660                 struct port_out_runtime *out = &p->out[port->id];
1661
1662                 out->pkt_tx = port->type->ops.pkt_tx;
1663                 out->flush = port->type->ops.flush;
1664                 out->obj = port->obj;
1665         }
1666
1667         return 0;
1668 }
1669
1670 static void
1671 port_out_build_free(struct rte_swx_pipeline *p)
1672 {
1673         free(p->out);
1674         p->out = NULL;
1675 }
1676
1677 static void
1678 port_out_free(struct rte_swx_pipeline *p)
1679 {
1680         port_out_build_free(p);
1681
1682         /* Output ports. */
1683         for ( ; ; ) {
1684                 struct port_out *port;
1685
1686                 port = TAILQ_FIRST(&p->ports_out);
1687                 if (!port)
1688                         break;
1689
1690                 TAILQ_REMOVE(&p->ports_out, port, node);
1691                 port->type->ops.free(port->obj);
1692                 free(port);
1693         }
1694
1695         /* Output port types. */
1696         for ( ; ; ) {
1697                 struct port_out_type *elem;
1698
1699                 elem = TAILQ_FIRST(&p->port_out_types);
1700                 if (!elem)
1701                         break;
1702
1703                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1704                 free(elem);
1705         }
1706 }
1707
1708 /*
1709  * Extern object.
1710  */
1711 static struct extern_type *
1712 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1713 {
1714         struct extern_type *elem;
1715
1716         TAILQ_FOREACH(elem, &p->extern_types, node)
1717                 if (strcmp(elem->name, name) == 0)
1718                         return elem;
1719
1720         return NULL;
1721 }
1722
1723 static struct extern_type_member_func *
1724 extern_type_member_func_find(struct extern_type *type, const char *name)
1725 {
1726         struct extern_type_member_func *elem;
1727
1728         TAILQ_FOREACH(elem, &type->funcs, node)
1729                 if (strcmp(elem->name, name) == 0)
1730                         return elem;
1731
1732         return NULL;
1733 }
1734
1735 static struct extern_obj *
1736 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1737 {
1738         struct extern_obj *elem;
1739
1740         TAILQ_FOREACH(elem, &p->extern_objs, node)
1741                 if (strcmp(elem->name, name) == 0)
1742                         return elem;
1743
1744         return NULL;
1745 }
1746
1747 static struct extern_type_member_func *
1748 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1749                              const char *name,
1750                              struct extern_obj **obj)
1751 {
1752         struct extern_obj *object;
1753         struct extern_type_member_func *func;
1754         char *object_name, *func_name;
1755
1756         if (name[0] != 'e' || name[1] != '.')
1757                 return NULL;
1758
1759         object_name = strdup(&name[2]);
1760         if (!object_name)
1761                 return NULL;
1762
1763         func_name = strchr(object_name, '.');
1764         if (!func_name) {
1765                 free(object_name);
1766                 return NULL;
1767         }
1768
1769         *func_name = 0;
1770         func_name++;
1771
1772         object = extern_obj_find(p, object_name);
1773         if (!object) {
1774                 free(object_name);
1775                 return NULL;
1776         }
1777
1778         func = extern_type_member_func_find(object->type, func_name);
1779         if (!func) {
1780                 free(object_name);
1781                 return NULL;
1782         }
1783
1784         if (obj)
1785                 *obj = object;
1786
1787         free(object_name);
1788         return func;
1789 }
1790
1791 static struct field *
1792 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1793                                const char *name,
1794                                struct extern_obj **object)
1795 {
1796         struct extern_obj *obj;
1797         struct field *f;
1798         char *obj_name, *field_name;
1799
1800         if ((name[0] != 'e') || (name[1] != '.'))
1801                 return NULL;
1802
1803         obj_name = strdup(&name[2]);
1804         if (!obj_name)
1805                 return NULL;
1806
1807         field_name = strchr(obj_name, '.');
1808         if (!field_name) {
1809                 free(obj_name);
1810                 return NULL;
1811         }
1812
1813         *field_name = 0;
1814         field_name++;
1815
1816         obj = extern_obj_find(p, obj_name);
1817         if (!obj) {
1818                 free(obj_name);
1819                 return NULL;
1820         }
1821
1822         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1823         if (!f) {
1824                 free(obj_name);
1825                 return NULL;
1826         }
1827
1828         if (object)
1829                 *object = obj;
1830
1831         free(obj_name);
1832         return f;
1833 }
1834
1835 int
1836 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1837         const char *name,
1838         const char *mailbox_struct_type_name,
1839         rte_swx_extern_type_constructor_t constructor,
1840         rte_swx_extern_type_destructor_t destructor)
1841 {
1842         struct extern_type *elem;
1843         struct struct_type *mailbox_struct_type;
1844
1845         CHECK(p, EINVAL);
1846
1847         CHECK_NAME(name, EINVAL);
1848         CHECK(!extern_type_find(p, name), EEXIST);
1849
1850         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1851         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1852         CHECK(mailbox_struct_type, EINVAL);
1853
1854         CHECK(constructor, EINVAL);
1855         CHECK(destructor, EINVAL);
1856
1857         /* Node allocation. */
1858         elem = calloc(1, sizeof(struct extern_type));
1859         CHECK(elem, ENOMEM);
1860
1861         /* Node initialization. */
1862         strcpy(elem->name, name);
1863         elem->mailbox_struct_type = mailbox_struct_type;
1864         elem->constructor = constructor;
1865         elem->destructor = destructor;
1866         TAILQ_INIT(&elem->funcs);
1867
1868         /* Node add to tailq. */
1869         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1870
1871         return 0;
1872 }
1873
1874 int
1875 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1876         const char *extern_type_name,
1877         const char *name,
1878         rte_swx_extern_type_member_func_t member_func)
1879 {
1880         struct extern_type *type;
1881         struct extern_type_member_func *type_member;
1882
1883         CHECK(p, EINVAL);
1884
1885         CHECK_NAME(extern_type_name, EINVAL);
1886         type = extern_type_find(p, extern_type_name);
1887         CHECK(type, EINVAL);
1888         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1889
1890         CHECK_NAME(name, EINVAL);
1891         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1892
1893         CHECK(member_func, EINVAL);
1894
1895         /* Node allocation. */
1896         type_member = calloc(1, sizeof(struct extern_type_member_func));
1897         CHECK(type_member, ENOMEM);
1898
1899         /* Node initialization. */
1900         strcpy(type_member->name, name);
1901         type_member->func = member_func;
1902         type_member->id = type->n_funcs;
1903
1904         /* Node add to tailq. */
1905         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1906         type->n_funcs++;
1907
1908         return 0;
1909 }
1910
1911 int
1912 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1913                                       const char *extern_type_name,
1914                                       const char *name,
1915                                       const char *args)
1916 {
1917         struct extern_type *type;
1918         struct extern_obj *obj;
1919         void *obj_handle;
1920
1921         CHECK(p, EINVAL);
1922
1923         CHECK_NAME(extern_type_name, EINVAL);
1924         type = extern_type_find(p, extern_type_name);
1925         CHECK(type, EINVAL);
1926
1927         CHECK_NAME(name, EINVAL);
1928         CHECK(!extern_obj_find(p, name), EEXIST);
1929
1930         /* Node allocation. */
1931         obj = calloc(1, sizeof(struct extern_obj));
1932         CHECK(obj, ENOMEM);
1933
1934         /* Object construction. */
1935         obj_handle = type->constructor(args);
1936         if (!obj_handle) {
1937                 free(obj);
1938                 CHECK(0, ENODEV);
1939         }
1940
1941         /* Node initialization. */
1942         strcpy(obj->name, name);
1943         obj->type = type;
1944         obj->obj = obj_handle;
1945         obj->struct_id = p->n_structs;
1946         obj->id = p->n_extern_objs;
1947
1948         /* Node add to tailq. */
1949         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1950         p->n_extern_objs++;
1951         p->n_structs++;
1952
1953         return 0;
1954 }
1955
1956 static int
1957 extern_obj_build(struct rte_swx_pipeline *p)
1958 {
1959         uint32_t i;
1960
1961         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1962                 struct thread *t = &p->threads[i];
1963                 struct extern_obj *obj;
1964
1965                 t->extern_objs = calloc(p->n_extern_objs,
1966                                         sizeof(struct extern_obj_runtime));
1967                 CHECK(t->extern_objs, ENOMEM);
1968
1969                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1970                         struct extern_obj_runtime *r =
1971                                 &t->extern_objs[obj->id];
1972                         struct extern_type_member_func *func;
1973                         uint32_t mailbox_size =
1974                                 obj->type->mailbox_struct_type->n_bits / 8;
1975
1976                         r->obj = obj->obj;
1977
1978                         r->mailbox = calloc(1, mailbox_size);
1979                         CHECK(r->mailbox, ENOMEM);
1980
1981                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1982                                 r->funcs[func->id] = func->func;
1983
1984                         t->structs[obj->struct_id] = r->mailbox;
1985                 }
1986         }
1987
1988         return 0;
1989 }
1990
1991 static void
1992 extern_obj_build_free(struct rte_swx_pipeline *p)
1993 {
1994         uint32_t i;
1995
1996         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1997                 struct thread *t = &p->threads[i];
1998                 uint32_t j;
1999
2000                 if (!t->extern_objs)
2001                         continue;
2002
2003                 for (j = 0; j < p->n_extern_objs; j++) {
2004                         struct extern_obj_runtime *r = &t->extern_objs[j];
2005
2006                         free(r->mailbox);
2007                 }
2008
2009                 free(t->extern_objs);
2010                 t->extern_objs = NULL;
2011         }
2012 }
2013
2014 static void
2015 extern_obj_free(struct rte_swx_pipeline *p)
2016 {
2017         extern_obj_build_free(p);
2018
2019         /* Extern objects. */
2020         for ( ; ; ) {
2021                 struct extern_obj *elem;
2022
2023                 elem = TAILQ_FIRST(&p->extern_objs);
2024                 if (!elem)
2025                         break;
2026
2027                 TAILQ_REMOVE(&p->extern_objs, elem, node);
2028                 if (elem->obj)
2029                         elem->type->destructor(elem->obj);
2030                 free(elem);
2031         }
2032
2033         /* Extern types. */
2034         for ( ; ; ) {
2035                 struct extern_type *elem;
2036
2037                 elem = TAILQ_FIRST(&p->extern_types);
2038                 if (!elem)
2039                         break;
2040
2041                 TAILQ_REMOVE(&p->extern_types, elem, node);
2042
2043                 for ( ; ; ) {
2044                         struct extern_type_member_func *func;
2045
2046                         func = TAILQ_FIRST(&elem->funcs);
2047                         if (!func)
2048                                 break;
2049
2050                         TAILQ_REMOVE(&elem->funcs, func, node);
2051                         free(func);
2052                 }
2053
2054                 free(elem);
2055         }
2056 }
2057
2058 /*
2059  * Extern function.
2060  */
2061 static struct extern_func *
2062 extern_func_find(struct rte_swx_pipeline *p, const char *name)
2063 {
2064         struct extern_func *elem;
2065
2066         TAILQ_FOREACH(elem, &p->extern_funcs, node)
2067                 if (strcmp(elem->name, name) == 0)
2068                         return elem;
2069
2070         return NULL;
2071 }
2072
2073 static struct extern_func *
2074 extern_func_parse(struct rte_swx_pipeline *p,
2075                   const char *name)
2076 {
2077         if (name[0] != 'f' || name[1] != '.')
2078                 return NULL;
2079
2080         return extern_func_find(p, &name[2]);
2081 }
2082
2083 static struct field *
2084 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
2085                                 const char *name,
2086                                 struct extern_func **function)
2087 {
2088         struct extern_func *func;
2089         struct field *f;
2090         char *func_name, *field_name;
2091
2092         if ((name[0] != 'f') || (name[1] != '.'))
2093                 return NULL;
2094
2095         func_name = strdup(&name[2]);
2096         if (!func_name)
2097                 return NULL;
2098
2099         field_name = strchr(func_name, '.');
2100         if (!field_name) {
2101                 free(func_name);
2102                 return NULL;
2103         }
2104
2105         *field_name = 0;
2106         field_name++;
2107
2108         func = extern_func_find(p, func_name);
2109         if (!func) {
2110                 free(func_name);
2111                 return NULL;
2112         }
2113
2114         f = struct_type_field_find(func->mailbox_struct_type, field_name);
2115         if (!f) {
2116                 free(func_name);
2117                 return NULL;
2118         }
2119
2120         if (function)
2121                 *function = func;
2122
2123         free(func_name);
2124         return f;
2125 }
2126
2127 int
2128 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
2129                                       const char *name,
2130                                       const char *mailbox_struct_type_name,
2131                                       rte_swx_extern_func_t func)
2132 {
2133         struct extern_func *f;
2134         struct struct_type *mailbox_struct_type;
2135
2136         CHECK(p, EINVAL);
2137
2138         CHECK_NAME(name, EINVAL);
2139         CHECK(!extern_func_find(p, name), EEXIST);
2140
2141         CHECK_NAME(mailbox_struct_type_name, EINVAL);
2142         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2143         CHECK(mailbox_struct_type, EINVAL);
2144
2145         CHECK(func, EINVAL);
2146
2147         /* Node allocation. */
2148         f = calloc(1, sizeof(struct extern_func));
2149         CHECK(func, ENOMEM);
2150
2151         /* Node initialization. */
2152         strcpy(f->name, name);
2153         f->mailbox_struct_type = mailbox_struct_type;
2154         f->func = func;
2155         f->struct_id = p->n_structs;
2156         f->id = p->n_extern_funcs;
2157
2158         /* Node add to tailq. */
2159         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
2160         p->n_extern_funcs++;
2161         p->n_structs++;
2162
2163         return 0;
2164 }
2165
2166 static int
2167 extern_func_build(struct rte_swx_pipeline *p)
2168 {
2169         uint32_t i;
2170
2171         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2172                 struct thread *t = &p->threads[i];
2173                 struct extern_func *func;
2174
2175                 /* Memory allocation. */
2176                 t->extern_funcs = calloc(p->n_extern_funcs,
2177                                          sizeof(struct extern_func_runtime));
2178                 CHECK(t->extern_funcs, ENOMEM);
2179
2180                 /* Extern function. */
2181                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
2182                         struct extern_func_runtime *r =
2183                                 &t->extern_funcs[func->id];
2184                         uint32_t mailbox_size =
2185                                 func->mailbox_struct_type->n_bits / 8;
2186
2187                         r->func = func->func;
2188
2189                         r->mailbox = calloc(1, mailbox_size);
2190                         CHECK(r->mailbox, ENOMEM);
2191
2192                         t->structs[func->struct_id] = r->mailbox;
2193                 }
2194         }
2195
2196         return 0;
2197 }
2198
2199 static void
2200 extern_func_build_free(struct rte_swx_pipeline *p)
2201 {
2202         uint32_t i;
2203
2204         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2205                 struct thread *t = &p->threads[i];
2206                 uint32_t j;
2207
2208                 if (!t->extern_funcs)
2209                         continue;
2210
2211                 for (j = 0; j < p->n_extern_funcs; j++) {
2212                         struct extern_func_runtime *r = &t->extern_funcs[j];
2213
2214                         free(r->mailbox);
2215                 }
2216
2217                 free(t->extern_funcs);
2218                 t->extern_funcs = NULL;
2219         }
2220 }
2221
2222 static void
2223 extern_func_free(struct rte_swx_pipeline *p)
2224 {
2225         extern_func_build_free(p);
2226
2227         for ( ; ; ) {
2228                 struct extern_func *elem;
2229
2230                 elem = TAILQ_FIRST(&p->extern_funcs);
2231                 if (!elem)
2232                         break;
2233
2234                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
2235                 free(elem);
2236         }
2237 }
2238
2239 /*
2240  * Header.
2241  */
2242 static struct header *
2243 header_find(struct rte_swx_pipeline *p, const char *name)
2244 {
2245         struct header *elem;
2246
2247         TAILQ_FOREACH(elem, &p->headers, node)
2248                 if (strcmp(elem->name, name) == 0)
2249                         return elem;
2250
2251         return NULL;
2252 }
2253
2254 static struct header *
2255 header_parse(struct rte_swx_pipeline *p,
2256              const char *name)
2257 {
2258         if (name[0] != 'h' || name[1] != '.')
2259                 return NULL;
2260
2261         return header_find(p, &name[2]);
2262 }
2263
2264 static struct field *
2265 header_field_parse(struct rte_swx_pipeline *p,
2266                    const char *name,
2267                    struct header **header)
2268 {
2269         struct header *h;
2270         struct field *f;
2271         char *header_name, *field_name;
2272
2273         if ((name[0] != 'h') || (name[1] != '.'))
2274                 return NULL;
2275
2276         header_name = strdup(&name[2]);
2277         if (!header_name)
2278                 return NULL;
2279
2280         field_name = strchr(header_name, '.');
2281         if (!field_name) {
2282                 free(header_name);
2283                 return NULL;
2284         }
2285
2286         *field_name = 0;
2287         field_name++;
2288
2289         h = header_find(p, header_name);
2290         if (!h) {
2291                 free(header_name);
2292                 return NULL;
2293         }
2294
2295         f = struct_type_field_find(h->st, field_name);
2296         if (!f) {
2297                 free(header_name);
2298                 return NULL;
2299         }
2300
2301         if (header)
2302                 *header = h;
2303
2304         free(header_name);
2305         return f;
2306 }
2307
2308 int
2309 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2310                                         const char *name,
2311                                         const char *struct_type_name)
2312 {
2313         struct struct_type *st;
2314         struct header *h;
2315         size_t n_headers_max;
2316
2317         CHECK(p, EINVAL);
2318         CHECK_NAME(name, EINVAL);
2319         CHECK_NAME(struct_type_name, EINVAL);
2320
2321         CHECK(!header_find(p, name), EEXIST);
2322
2323         st = struct_type_find(p, struct_type_name);
2324         CHECK(st, EINVAL);
2325
2326         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2327         CHECK(p->n_headers < n_headers_max, ENOSPC);
2328
2329         /* Node allocation. */
2330         h = calloc(1, sizeof(struct header));
2331         CHECK(h, ENOMEM);
2332
2333         /* Node initialization. */
2334         strcpy(h->name, name);
2335         h->st = st;
2336         h->struct_id = p->n_structs;
2337         h->id = p->n_headers;
2338
2339         /* Node add to tailq. */
2340         TAILQ_INSERT_TAIL(&p->headers, h, node);
2341         p->n_headers++;
2342         p->n_structs++;
2343
2344         return 0;
2345 }
2346
2347 static int
2348 header_build(struct rte_swx_pipeline *p)
2349 {
2350         struct header *h;
2351         uint32_t n_bytes = 0, i;
2352
2353         TAILQ_FOREACH(h, &p->headers, node) {
2354                 n_bytes += h->st->n_bits / 8;
2355         }
2356
2357         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2358                 struct thread *t = &p->threads[i];
2359                 uint32_t offset = 0;
2360
2361                 t->headers = calloc(p->n_headers,
2362                                     sizeof(struct header_runtime));
2363                 CHECK(t->headers, ENOMEM);
2364
2365                 t->headers_out = calloc(p->n_headers,
2366                                         sizeof(struct header_out_runtime));
2367                 CHECK(t->headers_out, ENOMEM);
2368
2369                 t->header_storage = calloc(1, n_bytes);
2370                 CHECK(t->header_storage, ENOMEM);
2371
2372                 t->header_out_storage = calloc(1, n_bytes);
2373                 CHECK(t->header_out_storage, ENOMEM);
2374
2375                 TAILQ_FOREACH(h, &p->headers, node) {
2376                         uint8_t *header_storage;
2377
2378                         header_storage = &t->header_storage[offset];
2379                         offset += h->st->n_bits / 8;
2380
2381                         t->headers[h->id].ptr0 = header_storage;
2382                         t->structs[h->struct_id] = header_storage;
2383                 }
2384         }
2385
2386         return 0;
2387 }
2388
2389 static void
2390 header_build_free(struct rte_swx_pipeline *p)
2391 {
2392         uint32_t i;
2393
2394         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2395                 struct thread *t = &p->threads[i];
2396
2397                 free(t->headers_out);
2398                 t->headers_out = NULL;
2399
2400                 free(t->headers);
2401                 t->headers = NULL;
2402
2403                 free(t->header_out_storage);
2404                 t->header_out_storage = NULL;
2405
2406                 free(t->header_storage);
2407                 t->header_storage = NULL;
2408         }
2409 }
2410
2411 static void
2412 header_free(struct rte_swx_pipeline *p)
2413 {
2414         header_build_free(p);
2415
2416         for ( ; ; ) {
2417                 struct header *elem;
2418
2419                 elem = TAILQ_FIRST(&p->headers);
2420                 if (!elem)
2421                         break;
2422
2423                 TAILQ_REMOVE(&p->headers, elem, node);
2424                 free(elem);
2425         }
2426 }
2427
2428 /*
2429  * Meta-data.
2430  */
2431 static struct field *
2432 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2433 {
2434         if (!p->metadata_st)
2435                 return NULL;
2436
2437         if (name[0] != 'm' || name[1] != '.')
2438                 return NULL;
2439
2440         return struct_type_field_find(p->metadata_st, &name[2]);
2441 }
2442
2443 int
2444 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2445                                           const char *struct_type_name)
2446 {
2447         struct struct_type *st = NULL;
2448
2449         CHECK(p, EINVAL);
2450
2451         CHECK_NAME(struct_type_name, EINVAL);
2452         st  = struct_type_find(p, struct_type_name);
2453         CHECK(st, EINVAL);
2454         CHECK(!p->metadata_st, EINVAL);
2455
2456         p->metadata_st = st;
2457         p->metadata_struct_id = p->n_structs;
2458
2459         p->n_structs++;
2460
2461         return 0;
2462 }
2463
2464 static int
2465 metadata_build(struct rte_swx_pipeline *p)
2466 {
2467         uint32_t n_bytes = p->metadata_st->n_bits / 8;
2468         uint32_t i;
2469
2470         /* Thread-level initialization. */
2471         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2472                 struct thread *t = &p->threads[i];
2473                 uint8_t *metadata;
2474
2475                 metadata = calloc(1, n_bytes);
2476                 CHECK(metadata, ENOMEM);
2477
2478                 t->metadata = metadata;
2479                 t->structs[p->metadata_struct_id] = metadata;
2480         }
2481
2482         return 0;
2483 }
2484
2485 static void
2486 metadata_build_free(struct rte_swx_pipeline *p)
2487 {
2488         uint32_t i;
2489
2490         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2491                 struct thread *t = &p->threads[i];
2492
2493                 free(t->metadata);
2494                 t->metadata = NULL;
2495         }
2496 }
2497
2498 static void
2499 metadata_free(struct rte_swx_pipeline *p)
2500 {
2501         metadata_build_free(p);
2502 }
2503
2504 /*
2505  * Instruction.
2506  */
2507 static int
2508 instruction_is_tx(enum instruction_type type)
2509 {
2510         switch (type) {
2511         case INSTR_TX:
2512         case INSTR_TX_I:
2513                 return 1;
2514
2515         default:
2516                 return 0;
2517         }
2518 }
2519
2520 static int
2521 instruction_is_jmp(struct instruction *instr)
2522 {
2523         switch (instr->type) {
2524         case INSTR_JMP:
2525         case INSTR_JMP_VALID:
2526         case INSTR_JMP_INVALID:
2527         case INSTR_JMP_HIT:
2528         case INSTR_JMP_MISS:
2529         case INSTR_JMP_ACTION_HIT:
2530         case INSTR_JMP_ACTION_MISS:
2531         case INSTR_JMP_EQ:
2532         case INSTR_JMP_EQ_S:
2533         case INSTR_JMP_EQ_I:
2534         case INSTR_JMP_NEQ:
2535         case INSTR_JMP_NEQ_S:
2536         case INSTR_JMP_NEQ_I:
2537         case INSTR_JMP_LT:
2538         case INSTR_JMP_LT_MH:
2539         case INSTR_JMP_LT_HM:
2540         case INSTR_JMP_LT_HH:
2541         case INSTR_JMP_LT_MI:
2542         case INSTR_JMP_LT_HI:
2543         case INSTR_JMP_GT:
2544         case INSTR_JMP_GT_MH:
2545         case INSTR_JMP_GT_HM:
2546         case INSTR_JMP_GT_HH:
2547         case INSTR_JMP_GT_MI:
2548         case INSTR_JMP_GT_HI:
2549                 return 1;
2550
2551         default:
2552                 return 0;
2553         }
2554 }
2555
2556 static struct field *
2557 action_field_parse(struct action *action, const char *name);
2558
2559 static struct field *
2560 struct_field_parse(struct rte_swx_pipeline *p,
2561                    struct action *action,
2562                    const char *name,
2563                    uint32_t *struct_id)
2564 {
2565         struct field *f;
2566
2567         switch (name[0]) {
2568         case 'h':
2569         {
2570                 struct header *header;
2571
2572                 f = header_field_parse(p, name, &header);
2573                 if (!f)
2574                         return NULL;
2575
2576                 *struct_id = header->struct_id;
2577                 return f;
2578         }
2579
2580         case 'm':
2581         {
2582                 f = metadata_field_parse(p, name);
2583                 if (!f)
2584                         return NULL;
2585
2586                 *struct_id = p->metadata_struct_id;
2587                 return f;
2588         }
2589
2590         case 't':
2591         {
2592                 if (!action)
2593                         return NULL;
2594
2595                 f = action_field_parse(action, name);
2596                 if (!f)
2597                         return NULL;
2598
2599                 *struct_id = 0;
2600                 return f;
2601         }
2602
2603         case 'e':
2604         {
2605                 struct extern_obj *obj;
2606
2607                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2608                 if (!f)
2609                         return NULL;
2610
2611                 *struct_id = obj->struct_id;
2612                 return f;
2613         }
2614
2615         case 'f':
2616         {
2617                 struct extern_func *func;
2618
2619                 f = extern_func_mailbox_field_parse(p, name, &func);
2620                 if (!f)
2621                         return NULL;
2622
2623                 *struct_id = func->struct_id;
2624                 return f;
2625         }
2626
2627         default:
2628                 return NULL;
2629         }
2630 }
2631
2632 static inline void
2633 pipeline_port_inc(struct rte_swx_pipeline *p)
2634 {
2635         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2636 }
2637
2638 static inline void
2639 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2640 {
2641         t->ip = p->instructions;
2642 }
2643
2644 static inline void
2645 thread_ip_set(struct thread *t, struct instruction *ip)
2646 {
2647         t->ip = ip;
2648 }
2649
2650 static inline void
2651 thread_ip_action_call(struct rte_swx_pipeline *p,
2652                       struct thread *t,
2653                       uint32_t action_id)
2654 {
2655         t->ret = t->ip + 1;
2656         t->ip = p->action_instructions[action_id];
2657 }
2658
2659 static inline void
2660 thread_ip_inc(struct rte_swx_pipeline *p);
2661
2662 static inline void
2663 thread_ip_inc(struct rte_swx_pipeline *p)
2664 {
2665         struct thread *t = &p->threads[p->thread_id];
2666
2667         t->ip++;
2668 }
2669
2670 static inline void
2671 thread_ip_inc_cond(struct thread *t, int cond)
2672 {
2673         t->ip += cond;
2674 }
2675
2676 static inline void
2677 thread_yield(struct rte_swx_pipeline *p)
2678 {
2679         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2680 }
2681
2682 static inline void
2683 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2684 {
2685         p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2686 }
2687
2688 /*
2689  * rx.
2690  */
2691 static int
2692 instr_rx_translate(struct rte_swx_pipeline *p,
2693                    struct action *action,
2694                    char **tokens,
2695                    int n_tokens,
2696                    struct instruction *instr,
2697                    struct instruction_data *data __rte_unused)
2698 {
2699         struct field *f;
2700
2701         CHECK(!action, EINVAL);
2702         CHECK(n_tokens == 2, EINVAL);
2703
2704         f = metadata_field_parse(p, tokens[1]);
2705         CHECK(f, EINVAL);
2706
2707         instr->type = INSTR_RX;
2708         instr->io.io.offset = f->offset / 8;
2709         instr->io.io.n_bits = f->n_bits;
2710         return 0;
2711 }
2712
2713 static inline void
2714 instr_rx_exec(struct rte_swx_pipeline *p);
2715
2716 static inline void
2717 instr_rx_exec(struct rte_swx_pipeline *p)
2718 {
2719         struct thread *t = &p->threads[p->thread_id];
2720         struct instruction *ip = t->ip;
2721         struct port_in_runtime *port = &p->in[p->port_id];
2722         struct rte_swx_pkt *pkt = &t->pkt;
2723         int pkt_received;
2724
2725         /* Packet. */
2726         pkt_received = port->pkt_rx(port->obj, pkt);
2727         t->ptr = &pkt->pkt[pkt->offset];
2728         rte_prefetch0(t->ptr);
2729
2730         TRACE("[Thread %2u] rx %s from port %u\n",
2731               p->thread_id,
2732               pkt_received ? "1 pkt" : "0 pkts",
2733               p->port_id);
2734
2735         /* Headers. */
2736         t->valid_headers = 0;
2737         t->n_headers_out = 0;
2738
2739         /* Meta-data. */
2740         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2741
2742         /* Tables. */
2743         t->table_state = p->table_state;
2744
2745         /* Thread. */
2746         pipeline_port_inc(p);
2747         thread_ip_inc_cond(t, pkt_received);
2748         thread_yield(p);
2749 }
2750
2751 /*
2752  * tx.
2753  */
2754 static int
2755 instr_tx_translate(struct rte_swx_pipeline *p,
2756                    struct action *action __rte_unused,
2757                    char **tokens,
2758                    int n_tokens,
2759                    struct instruction *instr,
2760                    struct instruction_data *data __rte_unused)
2761 {
2762         char *port = tokens[1];
2763         struct field *f;
2764         uint32_t port_val;
2765
2766         CHECK(n_tokens == 2, EINVAL);
2767
2768         f = metadata_field_parse(p, port);
2769         if (f) {
2770                 instr->type = INSTR_TX;
2771                 instr->io.io.offset = f->offset / 8;
2772                 instr->io.io.n_bits = f->n_bits;
2773                 return 0;
2774         }
2775
2776         /* TX_I. */
2777         port_val = strtoul(port, &port, 0);
2778         CHECK(!port[0], EINVAL);
2779
2780         instr->type = INSTR_TX_I;
2781         instr->io.io.val = port_val;
2782         return 0;
2783 }
2784
2785 static int
2786 instr_drop_translate(struct rte_swx_pipeline *p,
2787                      struct action *action __rte_unused,
2788                      char **tokens __rte_unused,
2789                      int n_tokens,
2790                      struct instruction *instr,
2791                      struct instruction_data *data __rte_unused)
2792 {
2793         CHECK(n_tokens == 1, EINVAL);
2794
2795         /* TX_I. */
2796         instr->type = INSTR_TX_I;
2797         instr->io.io.val = p->n_ports_out - 1;
2798         return 0;
2799 }
2800
2801 static inline void
2802 emit_handler(struct thread *t)
2803 {
2804         struct header_out_runtime *h0 = &t->headers_out[0];
2805         struct header_out_runtime *h1 = &t->headers_out[1];
2806         uint32_t offset = 0, i;
2807
2808         /* No header change or header decapsulation. */
2809         if ((t->n_headers_out == 1) &&
2810             (h0->ptr + h0->n_bytes == t->ptr)) {
2811                 TRACE("Emit handler: no header change or header decap.\n");
2812
2813                 t->pkt.offset -= h0->n_bytes;
2814                 t->pkt.length += h0->n_bytes;
2815
2816                 return;
2817         }
2818
2819         /* Header encapsulation (optionally, with prior header decasulation). */
2820         if ((t->n_headers_out == 2) &&
2821             (h1->ptr + h1->n_bytes == t->ptr) &&
2822             (h0->ptr == h0->ptr0)) {
2823                 uint32_t offset;
2824
2825                 TRACE("Emit handler: header encapsulation.\n");
2826
2827                 offset = h0->n_bytes + h1->n_bytes;
2828                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2829                 t->pkt.offset -= offset;
2830                 t->pkt.length += offset;
2831
2832                 return;
2833         }
2834
2835         /* Header insertion. */
2836         /* TBD */
2837
2838         /* Header extraction. */
2839         /* TBD */
2840
2841         /* For any other case. */
2842         TRACE("Emit handler: complex case.\n");
2843
2844         for (i = 0; i < t->n_headers_out; i++) {
2845                 struct header_out_runtime *h = &t->headers_out[i];
2846
2847                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2848                 offset += h->n_bytes;
2849         }
2850
2851         if (offset) {
2852                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2853                 t->pkt.offset -= offset;
2854                 t->pkt.length += offset;
2855         }
2856 }
2857
2858 static inline void
2859 instr_tx_exec(struct rte_swx_pipeline *p);
2860
2861 static inline void
2862 instr_tx_exec(struct rte_swx_pipeline *p)
2863 {
2864         struct thread *t = &p->threads[p->thread_id];
2865         struct instruction *ip = t->ip;
2866         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2867         struct port_out_runtime *port = &p->out[port_id];
2868         struct rte_swx_pkt *pkt = &t->pkt;
2869
2870         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2871               p->thread_id,
2872               (uint32_t)port_id);
2873
2874         /* Headers. */
2875         emit_handler(t);
2876
2877         /* Packet. */
2878         port->pkt_tx(port->obj, pkt);
2879
2880         /* Thread. */
2881         thread_ip_reset(p, t);
2882         instr_rx_exec(p);
2883 }
2884
2885 static inline void
2886 instr_tx_i_exec(struct rte_swx_pipeline *p)
2887 {
2888         struct thread *t = &p->threads[p->thread_id];
2889         struct instruction *ip = t->ip;
2890         uint64_t port_id = ip->io.io.val;
2891         struct port_out_runtime *port = &p->out[port_id];
2892         struct rte_swx_pkt *pkt = &t->pkt;
2893
2894         TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
2895               p->thread_id,
2896               (uint32_t)port_id);
2897
2898         /* Headers. */
2899         emit_handler(t);
2900
2901         /* Packet. */
2902         port->pkt_tx(port->obj, pkt);
2903
2904         /* Thread. */
2905         thread_ip_reset(p, t);
2906         instr_rx_exec(p);
2907 }
2908
2909 /*
2910  * extract.
2911  */
2912 static int
2913 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2914                             struct action *action,
2915                             char **tokens,
2916                             int n_tokens,
2917                             struct instruction *instr,
2918                             struct instruction_data *data __rte_unused)
2919 {
2920         struct header *h;
2921
2922         CHECK(!action, EINVAL);
2923         CHECK(n_tokens == 2, EINVAL);
2924
2925         h = header_parse(p, tokens[1]);
2926         CHECK(h, EINVAL);
2927
2928         instr->type = INSTR_HDR_EXTRACT;
2929         instr->io.hdr.header_id[0] = h->id;
2930         instr->io.hdr.struct_id[0] = h->struct_id;
2931         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2932         return 0;
2933 }
2934
2935 static inline void
2936 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2937
2938 static inline void
2939 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2940 {
2941         struct thread *t = &p->threads[p->thread_id];
2942         struct instruction *ip = t->ip;
2943         uint64_t valid_headers = t->valid_headers;
2944         uint8_t *ptr = t->ptr;
2945         uint32_t offset = t->pkt.offset;
2946         uint32_t length = t->pkt.length;
2947         uint32_t i;
2948
2949         for (i = 0; i < n_extract; i++) {
2950                 uint32_t header_id = ip->io.hdr.header_id[i];
2951                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2952                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2953
2954                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2955                       p->thread_id,
2956                       header_id,
2957                       n_bytes);
2958
2959                 /* Headers. */
2960                 t->structs[struct_id] = ptr;
2961                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2962
2963                 /* Packet. */
2964                 offset += n_bytes;
2965                 length -= n_bytes;
2966                 ptr += n_bytes;
2967         }
2968
2969         /* Headers. */
2970         t->valid_headers = valid_headers;
2971
2972         /* Packet. */
2973         t->pkt.offset = offset;
2974         t->pkt.length = length;
2975         t->ptr = ptr;
2976 }
2977
2978 static inline void
2979 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2980 {
2981         __instr_hdr_extract_exec(p, 1);
2982
2983         /* Thread. */
2984         thread_ip_inc(p);
2985 }
2986
2987 static inline void
2988 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2989 {
2990         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2991               p->thread_id);
2992
2993         __instr_hdr_extract_exec(p, 2);
2994
2995         /* Thread. */
2996         thread_ip_inc(p);
2997 }
2998
2999 static inline void
3000 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
3001 {
3002         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3003               p->thread_id);
3004
3005         __instr_hdr_extract_exec(p, 3);
3006
3007         /* Thread. */
3008         thread_ip_inc(p);
3009 }
3010
3011 static inline void
3012 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
3013 {
3014         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3015               p->thread_id);
3016
3017         __instr_hdr_extract_exec(p, 4);
3018
3019         /* Thread. */
3020         thread_ip_inc(p);
3021 }
3022
3023 static inline void
3024 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
3025 {
3026         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3027               p->thread_id);
3028
3029         __instr_hdr_extract_exec(p, 5);
3030
3031         /* Thread. */
3032         thread_ip_inc(p);
3033 }
3034
3035 static inline void
3036 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
3037 {
3038         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3039               p->thread_id);
3040
3041         __instr_hdr_extract_exec(p, 6);
3042
3043         /* Thread. */
3044         thread_ip_inc(p);
3045 }
3046
3047 static inline void
3048 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
3049 {
3050         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3051               p->thread_id);
3052
3053         __instr_hdr_extract_exec(p, 7);
3054
3055         /* Thread. */
3056         thread_ip_inc(p);
3057 }
3058
3059 static inline void
3060 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
3061 {
3062         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3063               p->thread_id);
3064
3065         __instr_hdr_extract_exec(p, 8);
3066
3067         /* Thread. */
3068         thread_ip_inc(p);
3069 }
3070
3071 /*
3072  * emit.
3073  */
3074 static int
3075 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
3076                          struct action *action __rte_unused,
3077                          char **tokens,
3078                          int n_tokens,
3079                          struct instruction *instr,
3080                          struct instruction_data *data __rte_unused)
3081 {
3082         struct header *h;
3083
3084         CHECK(n_tokens == 2, EINVAL);
3085
3086         h = header_parse(p, tokens[1]);
3087         CHECK(h, EINVAL);
3088
3089         instr->type = INSTR_HDR_EMIT;
3090         instr->io.hdr.header_id[0] = h->id;
3091         instr->io.hdr.struct_id[0] = h->struct_id;
3092         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3093         return 0;
3094 }
3095
3096 static inline void
3097 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
3098
3099 static inline void
3100 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
3101 {
3102         struct thread *t = &p->threads[p->thread_id];
3103         struct instruction *ip = t->ip;
3104         uint32_t n_headers_out = t->n_headers_out;
3105         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
3106         uint8_t *ho_ptr = NULL;
3107         uint32_t ho_nbytes = 0, i;
3108
3109         for (i = 0; i < n_emit; i++) {
3110                 uint32_t header_id = ip->io.hdr.header_id[i];
3111                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3112                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3113
3114                 struct header_runtime *hi = &t->headers[header_id];
3115                 uint8_t *hi_ptr = t->structs[struct_id];
3116
3117                 TRACE("[Thread %2u]: emit header %u\n",
3118                       p->thread_id,
3119                       header_id);
3120
3121                 /* Headers. */
3122                 if (!i) {
3123                         if (!t->n_headers_out) {
3124                                 ho = &t->headers_out[0];
3125
3126                                 ho->ptr0 = hi->ptr0;
3127                                 ho->ptr = hi_ptr;
3128
3129                                 ho_ptr = hi_ptr;
3130                                 ho_nbytes = n_bytes;
3131
3132                                 n_headers_out = 1;
3133
3134                                 continue;
3135                         } else {
3136                                 ho_ptr = ho->ptr;
3137                                 ho_nbytes = ho->n_bytes;
3138                         }
3139                 }
3140
3141                 if (ho_ptr + ho_nbytes == hi_ptr) {
3142                         ho_nbytes += n_bytes;
3143                 } else {
3144                         ho->n_bytes = ho_nbytes;
3145
3146                         ho++;
3147                         ho->ptr0 = hi->ptr0;
3148                         ho->ptr = hi_ptr;
3149
3150                         ho_ptr = hi_ptr;
3151                         ho_nbytes = n_bytes;
3152
3153                         n_headers_out++;
3154                 }
3155         }
3156
3157         ho->n_bytes = ho_nbytes;
3158         t->n_headers_out = n_headers_out;
3159 }
3160
3161 static inline void
3162 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
3163 {
3164         __instr_hdr_emit_exec(p, 1);
3165
3166         /* Thread. */
3167         thread_ip_inc(p);
3168 }
3169
3170 static inline void
3171 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
3172 {
3173         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3174               p->thread_id);
3175
3176         __instr_hdr_emit_exec(p, 1);
3177         instr_tx_exec(p);
3178 }
3179
3180 static inline void
3181 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
3182 {
3183         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3184               p->thread_id);
3185
3186         __instr_hdr_emit_exec(p, 2);
3187         instr_tx_exec(p);
3188 }
3189
3190 static inline void
3191 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
3192 {
3193         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3194               p->thread_id);
3195
3196         __instr_hdr_emit_exec(p, 3);
3197         instr_tx_exec(p);
3198 }
3199
3200 static inline void
3201 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
3202 {
3203         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3204               p->thread_id);
3205
3206         __instr_hdr_emit_exec(p, 4);
3207         instr_tx_exec(p);
3208 }
3209
3210 static inline void
3211 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
3212 {
3213         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3214               p->thread_id);
3215
3216         __instr_hdr_emit_exec(p, 5);
3217         instr_tx_exec(p);
3218 }
3219
3220 static inline void
3221 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
3222 {
3223         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3224               p->thread_id);
3225
3226         __instr_hdr_emit_exec(p, 6);
3227         instr_tx_exec(p);
3228 }
3229
3230 static inline void
3231 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
3232 {
3233         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3234               p->thread_id);
3235
3236         __instr_hdr_emit_exec(p, 7);
3237         instr_tx_exec(p);
3238 }
3239
3240 static inline void
3241 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
3242 {
3243         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
3244               p->thread_id);
3245
3246         __instr_hdr_emit_exec(p, 8);
3247         instr_tx_exec(p);
3248 }
3249
3250 /*
3251  * validate.
3252  */
3253 static int
3254 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
3255                              struct action *action __rte_unused,
3256                              char **tokens,
3257                              int n_tokens,
3258                              struct instruction *instr,
3259                              struct instruction_data *data __rte_unused)
3260 {
3261         struct header *h;
3262
3263         CHECK(n_tokens == 2, EINVAL);
3264
3265         h = header_parse(p, tokens[1]);
3266         CHECK(h, EINVAL);
3267
3268         instr->type = INSTR_HDR_VALIDATE;
3269         instr->valid.header_id = h->id;
3270         return 0;
3271 }
3272
3273 static inline void
3274 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
3275 {
3276         struct thread *t = &p->threads[p->thread_id];
3277         struct instruction *ip = t->ip;
3278         uint32_t header_id = ip->valid.header_id;
3279
3280         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
3281
3282         /* Headers. */
3283         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
3284
3285         /* Thread. */
3286         thread_ip_inc(p);
3287 }
3288
3289 /*
3290  * invalidate.
3291  */
3292 static int
3293 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
3294                                struct action *action __rte_unused,
3295                                char **tokens,
3296                                int n_tokens,
3297                                struct instruction *instr,
3298                                struct instruction_data *data __rte_unused)
3299 {
3300         struct header *h;
3301
3302         CHECK(n_tokens == 2, EINVAL);
3303
3304         h = header_parse(p, tokens[1]);
3305         CHECK(h, EINVAL);
3306
3307         instr->type = INSTR_HDR_INVALIDATE;
3308         instr->valid.header_id = h->id;
3309         return 0;
3310 }
3311
3312 static inline void
3313 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3314 {
3315         struct thread *t = &p->threads[p->thread_id];
3316         struct instruction *ip = t->ip;
3317         uint32_t header_id = ip->valid.header_id;
3318
3319         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3320
3321         /* Headers. */
3322         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3323
3324         /* Thread. */
3325         thread_ip_inc(p);
3326 }
3327
3328 /*
3329  * table.
3330  */
3331 static struct table *
3332 table_find(struct rte_swx_pipeline *p, const char *name);
3333
3334 static int
3335 instr_table_translate(struct rte_swx_pipeline *p,
3336                       struct action *action,
3337                       char **tokens,
3338                       int n_tokens,
3339                       struct instruction *instr,
3340                       struct instruction_data *data __rte_unused)
3341 {
3342         struct table *t;
3343
3344         CHECK(!action, EINVAL);
3345         CHECK(n_tokens == 2, EINVAL);
3346
3347         t = table_find(p, tokens[1]);
3348         CHECK(t, EINVAL);
3349
3350         instr->type = INSTR_TABLE;
3351         instr->table.table_id = t->id;
3352         return 0;
3353 }
3354
3355 static inline void
3356 instr_table_exec(struct rte_swx_pipeline *p)
3357 {
3358         struct thread *t = &p->threads[p->thread_id];
3359         struct instruction *ip = t->ip;
3360         uint32_t table_id = ip->table.table_id;
3361         struct rte_swx_table_state *ts = &t->table_state[table_id];
3362         struct table_runtime *table = &t->tables[table_id];
3363         struct table_statistics *stats = &p->table_stats[table_id];
3364         uint64_t action_id, n_pkts_hit, n_pkts_action;
3365         uint8_t *action_data;
3366         int done, hit;
3367
3368         /* Table. */
3369         done = table->func(ts->obj,
3370                            table->mailbox,
3371                            table->key,
3372                            &action_id,
3373                            &action_data,
3374                            &hit);
3375         if (!done) {
3376                 /* Thread. */
3377                 TRACE("[Thread %2u] table %u (not finalized)\n",
3378                       p->thread_id,
3379                       table_id);
3380
3381                 thread_yield(p);
3382                 return;
3383         }
3384
3385         action_id = hit ? action_id : ts->default_action_id;
3386         action_data = hit ? action_data : ts->default_action_data;
3387         n_pkts_hit = stats->n_pkts_hit[hit];
3388         n_pkts_action = stats->n_pkts_action[action_id];
3389
3390         TRACE("[Thread %2u] table %u (%s, action %u)\n",
3391               p->thread_id,
3392               table_id,
3393               hit ? "hit" : "miss",
3394               (uint32_t)action_id);
3395
3396         t->action_id = action_id;
3397         t->structs[0] = action_data;
3398         t->hit = hit;
3399         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
3400         stats->n_pkts_action[action_id] = n_pkts_action + 1;
3401
3402         /* Thread. */
3403         thread_ip_action_call(p, t, action_id);
3404 }
3405
3406 /*
3407  * extern.
3408  */
3409 static int
3410 instr_extern_translate(struct rte_swx_pipeline *p,
3411                        struct action *action __rte_unused,
3412                        char **tokens,
3413                        int n_tokens,
3414                        struct instruction *instr,
3415                        struct instruction_data *data __rte_unused)
3416 {
3417         char *token = tokens[1];
3418
3419         CHECK(n_tokens == 2, EINVAL);
3420
3421         if (token[0] == 'e') {
3422                 struct extern_obj *obj;
3423                 struct extern_type_member_func *func;
3424
3425                 func = extern_obj_member_func_parse(p, token, &obj);
3426                 CHECK(func, EINVAL);
3427
3428                 instr->type = INSTR_EXTERN_OBJ;
3429                 instr->ext_obj.ext_obj_id = obj->id;
3430                 instr->ext_obj.func_id = func->id;
3431
3432                 return 0;
3433         }
3434
3435         if (token[0] == 'f') {
3436                 struct extern_func *func;
3437
3438                 func = extern_func_parse(p, token);
3439                 CHECK(func, EINVAL);
3440
3441                 instr->type = INSTR_EXTERN_FUNC;
3442                 instr->ext_func.ext_func_id = func->id;
3443
3444                 return 0;
3445         }
3446
3447         CHECK(0, EINVAL);
3448 }
3449
3450 static inline void
3451 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3452 {
3453         struct thread *t = &p->threads[p->thread_id];
3454         struct instruction *ip = t->ip;
3455         uint32_t obj_id = ip->ext_obj.ext_obj_id;
3456         uint32_t func_id = ip->ext_obj.func_id;
3457         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3458         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3459
3460         TRACE("[Thread %2u] extern obj %u member func %u\n",
3461               p->thread_id,
3462               obj_id,
3463               func_id);
3464
3465         /* Extern object member function execute. */
3466         uint32_t done = func(obj->obj, obj->mailbox);
3467
3468         /* Thread. */
3469         thread_ip_inc_cond(t, done);
3470         thread_yield_cond(p, done ^ 1);
3471 }
3472
3473 static inline void
3474 instr_extern_func_exec(struct rte_swx_pipeline *p)
3475 {
3476         struct thread *t = &p->threads[p->thread_id];
3477         struct instruction *ip = t->ip;
3478         uint32_t ext_func_id = ip->ext_func.ext_func_id;
3479         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3480         rte_swx_extern_func_t func = ext_func->func;
3481
3482         TRACE("[Thread %2u] extern func %u\n",
3483               p->thread_id,
3484               ext_func_id);
3485
3486         /* Extern function execute. */
3487         uint32_t done = func(ext_func->mailbox);
3488
3489         /* Thread. */
3490         thread_ip_inc_cond(t, done);
3491         thread_yield_cond(p, done ^ 1);
3492 }
3493
3494 /*
3495  * mov.
3496  */
3497 static int
3498 instr_mov_translate(struct rte_swx_pipeline *p,
3499                     struct action *action,
3500                     char **tokens,
3501                     int n_tokens,
3502                     struct instruction *instr,
3503                     struct instruction_data *data __rte_unused)
3504 {
3505         char *dst = tokens[1], *src = tokens[2];
3506         struct field *fdst, *fsrc;
3507         uint64_t src_val;
3508         uint32_t dst_struct_id, src_struct_id;
3509
3510         CHECK(n_tokens == 3, EINVAL);
3511
3512         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3513         CHECK(fdst, EINVAL);
3514
3515         /* MOV or MOV_S. */
3516         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3517         if (fsrc) {
3518                 instr->type = INSTR_MOV;
3519                 if ((dst[0] == 'h' && src[0] != 'h') ||
3520                     (dst[0] != 'h' && src[0] == 'h'))
3521                         instr->type = INSTR_MOV_S;
3522
3523                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3524                 instr->mov.dst.n_bits = fdst->n_bits;
3525                 instr->mov.dst.offset = fdst->offset / 8;
3526                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3527                 instr->mov.src.n_bits = fsrc->n_bits;
3528                 instr->mov.src.offset = fsrc->offset / 8;
3529                 return 0;
3530         }
3531
3532         /* MOV_I. */
3533         src_val = strtoull(src, &src, 0);
3534         CHECK(!src[0], EINVAL);
3535
3536         if (dst[0] == 'h')
3537                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3538
3539         instr->type = INSTR_MOV_I;
3540         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3541         instr->mov.dst.n_bits = fdst->n_bits;
3542         instr->mov.dst.offset = fdst->offset / 8;
3543         instr->mov.src_val = src_val;
3544         return 0;
3545 }
3546
3547 static inline void
3548 instr_mov_exec(struct rte_swx_pipeline *p)
3549 {
3550         struct thread *t = &p->threads[p->thread_id];
3551         struct instruction *ip = t->ip;
3552
3553         TRACE("[Thread %2u] mov\n",
3554               p->thread_id);
3555
3556         MOV(t, ip);
3557
3558         /* Thread. */
3559         thread_ip_inc(p);
3560 }
3561
3562 static inline void
3563 instr_mov_s_exec(struct rte_swx_pipeline *p)
3564 {
3565         struct thread *t = &p->threads[p->thread_id];
3566         struct instruction *ip = t->ip;
3567
3568         TRACE("[Thread %2u] mov (s)\n",
3569               p->thread_id);
3570
3571         MOV_S(t, ip);
3572
3573         /* Thread. */
3574         thread_ip_inc(p);
3575 }
3576
3577 static inline void
3578 instr_mov_i_exec(struct rte_swx_pipeline *p)
3579 {
3580         struct thread *t = &p->threads[p->thread_id];
3581         struct instruction *ip = t->ip;
3582
3583         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
3584               p->thread_id,
3585               ip->mov.src_val);
3586
3587         MOV_I(t, ip);
3588
3589         /* Thread. */
3590         thread_ip_inc(p);
3591 }
3592
3593 /*
3594  * dma.
3595  */
3596 static int
3597 instr_dma_translate(struct rte_swx_pipeline *p,
3598                     struct action *action,
3599                     char **tokens,
3600                     int n_tokens,
3601                     struct instruction *instr,
3602                     struct instruction_data *data __rte_unused)
3603 {
3604         char *dst = tokens[1];
3605         char *src = tokens[2];
3606         struct header *h;
3607         struct field *tf;
3608
3609         CHECK(action, EINVAL);
3610         CHECK(n_tokens == 3, EINVAL);
3611
3612         h = header_parse(p, dst);
3613         CHECK(h, EINVAL);
3614
3615         tf = action_field_parse(action, src);
3616         CHECK(tf, EINVAL);
3617
3618         instr->type = INSTR_DMA_HT;
3619         instr->dma.dst.header_id[0] = h->id;
3620         instr->dma.dst.struct_id[0] = h->struct_id;
3621         instr->dma.n_bytes[0] = h->st->n_bits / 8;
3622         instr->dma.src.offset[0] = tf->offset / 8;
3623
3624         return 0;
3625 }
3626
3627 static inline void
3628 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3629
3630 static inline void
3631 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3632 {
3633         struct thread *t = &p->threads[p->thread_id];
3634         struct instruction *ip = t->ip;
3635         uint8_t *action_data = t->structs[0];
3636         uint64_t valid_headers = t->valid_headers;
3637         uint32_t i;
3638
3639         for (i = 0; i < n_dma; i++) {
3640                 uint32_t header_id = ip->dma.dst.header_id[i];
3641                 uint32_t struct_id = ip->dma.dst.struct_id[i];
3642                 uint32_t offset = ip->dma.src.offset[i];
3643                 uint32_t n_bytes = ip->dma.n_bytes[i];
3644
3645                 struct header_runtime *h = &t->headers[header_id];
3646                 uint8_t *h_ptr0 = h->ptr0;
3647                 uint8_t *h_ptr = t->structs[struct_id];
3648
3649                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3650                         h_ptr : h_ptr0;
3651                 void *src = &action_data[offset];
3652
3653                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3654
3655                 /* Headers. */
3656                 memcpy(dst, src, n_bytes);
3657                 t->structs[struct_id] = dst;
3658                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3659         }
3660
3661         t->valid_headers = valid_headers;
3662 }
3663
3664 static inline void
3665 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3666 {
3667         __instr_dma_ht_exec(p, 1);
3668
3669         /* Thread. */
3670         thread_ip_inc(p);
3671 }
3672
3673 static inline void
3674 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3675 {
3676         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3677               p->thread_id);
3678
3679         __instr_dma_ht_exec(p, 2);
3680
3681         /* Thread. */
3682         thread_ip_inc(p);
3683 }
3684
3685 static inline void
3686 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3687 {
3688         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3689               p->thread_id);
3690
3691         __instr_dma_ht_exec(p, 3);
3692
3693         /* Thread. */
3694         thread_ip_inc(p);
3695 }
3696
3697 static inline void
3698 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3699 {
3700         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3701               p->thread_id);
3702
3703         __instr_dma_ht_exec(p, 4);
3704
3705         /* Thread. */
3706         thread_ip_inc(p);
3707 }
3708
3709 static inline void
3710 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3711 {
3712         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3713               p->thread_id);
3714
3715         __instr_dma_ht_exec(p, 5);
3716
3717         /* Thread. */
3718         thread_ip_inc(p);
3719 }
3720
3721 static inline void
3722 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3723 {
3724         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3725               p->thread_id);
3726
3727         __instr_dma_ht_exec(p, 6);
3728
3729         /* Thread. */
3730         thread_ip_inc(p);
3731 }
3732
3733 static inline void
3734 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3735 {
3736         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3737               p->thread_id);
3738
3739         __instr_dma_ht_exec(p, 7);
3740
3741         /* Thread. */
3742         thread_ip_inc(p);
3743 }
3744
3745 static inline void
3746 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3747 {
3748         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3749               p->thread_id);
3750
3751         __instr_dma_ht_exec(p, 8);
3752
3753         /* Thread. */
3754         thread_ip_inc(p);
3755 }
3756
3757 /*
3758  * alu.
3759  */
3760 static int
3761 instr_alu_add_translate(struct rte_swx_pipeline *p,
3762                         struct action *action,
3763                         char **tokens,
3764                         int n_tokens,
3765                         struct instruction *instr,
3766                         struct instruction_data *data __rte_unused)
3767 {
3768         char *dst = tokens[1], *src = tokens[2];
3769         struct field *fdst, *fsrc;
3770         uint64_t src_val;
3771         uint32_t dst_struct_id, src_struct_id;
3772
3773         CHECK(n_tokens == 3, EINVAL);
3774
3775         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3776         CHECK(fdst, EINVAL);
3777
3778         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3779         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3780         if (fsrc) {
3781                 instr->type = INSTR_ALU_ADD;
3782                 if (dst[0] == 'h' && src[0] != 'h')
3783                         instr->type = INSTR_ALU_ADD_HM;
3784                 if (dst[0] != 'h' && src[0] == 'h')
3785                         instr->type = INSTR_ALU_ADD_MH;
3786                 if (dst[0] == 'h' && src[0] == 'h')
3787                         instr->type = INSTR_ALU_ADD_HH;
3788
3789                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3790                 instr->alu.dst.n_bits = fdst->n_bits;
3791                 instr->alu.dst.offset = fdst->offset / 8;
3792                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3793                 instr->alu.src.n_bits = fsrc->n_bits;
3794                 instr->alu.src.offset = fsrc->offset / 8;
3795                 return 0;
3796         }
3797
3798         /* ADD_MI, ADD_HI. */
3799         src_val = strtoull(src, &src, 0);
3800         CHECK(!src[0], EINVAL);
3801
3802         instr->type = INSTR_ALU_ADD_MI;
3803         if (dst[0] == 'h')
3804                 instr->type = INSTR_ALU_ADD_HI;
3805
3806         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3807         instr->alu.dst.n_bits = fdst->n_bits;
3808         instr->alu.dst.offset = fdst->offset / 8;
3809         instr->alu.src_val = src_val;
3810         return 0;
3811 }
3812
3813 static int
3814 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3815                         struct action *action,
3816                         char **tokens,
3817                         int n_tokens,
3818                         struct instruction *instr,
3819                         struct instruction_data *data __rte_unused)
3820 {
3821         char *dst = tokens[1], *src = tokens[2];
3822         struct field *fdst, *fsrc;
3823         uint64_t src_val;
3824         uint32_t dst_struct_id, src_struct_id;
3825
3826         CHECK(n_tokens == 3, EINVAL);
3827
3828         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3829         CHECK(fdst, EINVAL);
3830
3831         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3832         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3833         if (fsrc) {
3834                 instr->type = INSTR_ALU_SUB;
3835                 if (dst[0] == 'h' && src[0] != 'h')
3836                         instr->type = INSTR_ALU_SUB_HM;
3837                 if (dst[0] != 'h' && src[0] == 'h')
3838                         instr->type = INSTR_ALU_SUB_MH;
3839                 if (dst[0] == 'h' && src[0] == 'h')
3840                         instr->type = INSTR_ALU_SUB_HH;
3841
3842                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3843                 instr->alu.dst.n_bits = fdst->n_bits;
3844                 instr->alu.dst.offset = fdst->offset / 8;
3845                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3846                 instr->alu.src.n_bits = fsrc->n_bits;
3847                 instr->alu.src.offset = fsrc->offset / 8;
3848                 return 0;
3849         }
3850
3851         /* SUB_MI, SUB_HI. */
3852         src_val = strtoull(src, &src, 0);
3853         CHECK(!src[0], EINVAL);
3854
3855         instr->type = INSTR_ALU_SUB_MI;
3856         if (dst[0] == 'h')
3857                 instr->type = INSTR_ALU_SUB_HI;
3858
3859         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3860         instr->alu.dst.n_bits = fdst->n_bits;
3861         instr->alu.dst.offset = fdst->offset / 8;
3862         instr->alu.src_val = src_val;
3863         return 0;
3864 }
3865
3866 static int
3867 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3868                           struct action *action __rte_unused,
3869                           char **tokens,
3870                           int n_tokens,
3871                           struct instruction *instr,
3872                           struct instruction_data *data __rte_unused)
3873 {
3874         char *dst = tokens[1], *src = tokens[2];
3875         struct header *hdst, *hsrc;
3876         struct field *fdst, *fsrc;
3877
3878         CHECK(n_tokens == 3, EINVAL);
3879
3880         fdst = header_field_parse(p, dst, &hdst);
3881         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3882
3883         /* CKADD_FIELD. */
3884         fsrc = header_field_parse(p, src, &hsrc);
3885         if (fsrc) {
3886                 instr->type = INSTR_ALU_CKADD_FIELD;
3887                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3888                 instr->alu.dst.n_bits = fdst->n_bits;
3889                 instr->alu.dst.offset = fdst->offset / 8;
3890                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3891                 instr->alu.src.n_bits = fsrc->n_bits;
3892                 instr->alu.src.offset = fsrc->offset / 8;
3893                 return 0;
3894         }
3895
3896         /* CKADD_STRUCT, CKADD_STRUCT20. */
3897         hsrc = header_parse(p, src);
3898         CHECK(hsrc, EINVAL);
3899
3900         instr->type = INSTR_ALU_CKADD_STRUCT;
3901         if ((hsrc->st->n_bits / 8) == 20)
3902                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3903
3904         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3905         instr->alu.dst.n_bits = fdst->n_bits;
3906         instr->alu.dst.offset = fdst->offset / 8;
3907         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3908         instr->alu.src.n_bits = hsrc->st->n_bits;
3909         instr->alu.src.offset = 0; /* Unused. */
3910         return 0;
3911 }
3912
3913 static int
3914 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3915                           struct action *action __rte_unused,
3916                           char **tokens,
3917                           int n_tokens,
3918                           struct instruction *instr,
3919                           struct instruction_data *data __rte_unused)
3920 {
3921         char *dst = tokens[1], *src = tokens[2];
3922         struct header *hdst, *hsrc;
3923         struct field *fdst, *fsrc;
3924
3925         CHECK(n_tokens == 3, EINVAL);
3926
3927         fdst = header_field_parse(p, dst, &hdst);
3928         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3929
3930         fsrc = header_field_parse(p, src, &hsrc);
3931         CHECK(fsrc, EINVAL);
3932
3933         instr->type = INSTR_ALU_CKSUB_FIELD;
3934         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3935         instr->alu.dst.n_bits = fdst->n_bits;
3936         instr->alu.dst.offset = fdst->offset / 8;
3937         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3938         instr->alu.src.n_bits = fsrc->n_bits;
3939         instr->alu.src.offset = fsrc->offset / 8;
3940         return 0;
3941 }
3942
3943 static int
3944 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3945                         struct action *action,
3946                         char **tokens,
3947                         int n_tokens,
3948                         struct instruction *instr,
3949                         struct instruction_data *data __rte_unused)
3950 {
3951         char *dst = tokens[1], *src = tokens[2];
3952         struct field *fdst, *fsrc;
3953         uint64_t src_val;
3954         uint32_t dst_struct_id, src_struct_id;
3955
3956         CHECK(n_tokens == 3, EINVAL);
3957
3958         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3959         CHECK(fdst, EINVAL);
3960
3961         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3962         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3963         if (fsrc) {
3964                 instr->type = INSTR_ALU_SHL;
3965                 if (dst[0] == 'h' && src[0] != 'h')
3966                         instr->type = INSTR_ALU_SHL_HM;
3967                 if (dst[0] != 'h' && src[0] == 'h')
3968                         instr->type = INSTR_ALU_SHL_MH;
3969                 if (dst[0] == 'h' && src[0] == 'h')
3970                         instr->type = INSTR_ALU_SHL_HH;
3971
3972                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3973                 instr->alu.dst.n_bits = fdst->n_bits;
3974                 instr->alu.dst.offset = fdst->offset / 8;
3975                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3976                 instr->alu.src.n_bits = fsrc->n_bits;
3977                 instr->alu.src.offset = fsrc->offset / 8;
3978                 return 0;
3979         }
3980
3981         /* SHL_MI, SHL_HI. */
3982         src_val = strtoull(src, &src, 0);
3983         CHECK(!src[0], EINVAL);
3984
3985         instr->type = INSTR_ALU_SHL_MI;
3986         if (dst[0] == 'h')
3987                 instr->type = INSTR_ALU_SHL_HI;
3988
3989         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3990         instr->alu.dst.n_bits = fdst->n_bits;
3991         instr->alu.dst.offset = fdst->offset / 8;
3992         instr->alu.src_val = src_val;
3993         return 0;
3994 }
3995
3996 static int
3997 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3998                         struct action *action,
3999                         char **tokens,
4000                         int n_tokens,
4001                         struct instruction *instr,
4002                         struct instruction_data *data __rte_unused)
4003 {
4004         char *dst = tokens[1], *src = tokens[2];
4005         struct field *fdst, *fsrc;
4006         uint64_t src_val;
4007         uint32_t dst_struct_id, src_struct_id;
4008
4009         CHECK(n_tokens == 3, EINVAL);
4010
4011         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4012         CHECK(fdst, EINVAL);
4013
4014         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
4015         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4016         if (fsrc) {
4017                 instr->type = INSTR_ALU_SHR;
4018                 if (dst[0] == 'h' && src[0] != 'h')
4019                         instr->type = INSTR_ALU_SHR_HM;
4020                 if (dst[0] != 'h' && src[0] == 'h')
4021                         instr->type = INSTR_ALU_SHR_MH;
4022                 if (dst[0] == 'h' && src[0] == 'h')
4023                         instr->type = INSTR_ALU_SHR_HH;
4024
4025                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4026                 instr->alu.dst.n_bits = fdst->n_bits;
4027                 instr->alu.dst.offset = fdst->offset / 8;
4028                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4029                 instr->alu.src.n_bits = fsrc->n_bits;
4030                 instr->alu.src.offset = fsrc->offset / 8;
4031                 return 0;
4032         }
4033
4034         /* SHR_MI, SHR_HI. */
4035         src_val = strtoull(src, &src, 0);
4036         CHECK(!src[0], EINVAL);
4037
4038         instr->type = INSTR_ALU_SHR_MI;
4039         if (dst[0] == 'h')
4040                 instr->type = INSTR_ALU_SHR_HI;
4041
4042         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4043         instr->alu.dst.n_bits = fdst->n_bits;
4044         instr->alu.dst.offset = fdst->offset / 8;
4045         instr->alu.src_val = src_val;
4046         return 0;
4047 }
4048
4049 static int
4050 instr_alu_and_translate(struct rte_swx_pipeline *p,
4051                         struct action *action,
4052                         char **tokens,
4053                         int n_tokens,
4054                         struct instruction *instr,
4055                         struct instruction_data *data __rte_unused)
4056 {
4057         char *dst = tokens[1], *src = tokens[2];
4058         struct field *fdst, *fsrc;
4059         uint64_t src_val;
4060         uint32_t dst_struct_id, src_struct_id;
4061
4062         CHECK(n_tokens == 3, EINVAL);
4063
4064         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4065         CHECK(fdst, EINVAL);
4066
4067         /* AND or AND_S. */
4068         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4069         if (fsrc) {
4070                 instr->type = INSTR_ALU_AND;
4071                 if ((dst[0] == 'h' && src[0] != 'h') ||
4072                     (dst[0] != 'h' && src[0] == 'h'))
4073                         instr->type = INSTR_ALU_AND_S;
4074
4075                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4076                 instr->alu.dst.n_bits = fdst->n_bits;
4077                 instr->alu.dst.offset = fdst->offset / 8;
4078                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4079                 instr->alu.src.n_bits = fsrc->n_bits;
4080                 instr->alu.src.offset = fsrc->offset / 8;
4081                 return 0;
4082         }
4083
4084         /* AND_I. */
4085         src_val = strtoull(src, &src, 0);
4086         CHECK(!src[0], EINVAL);
4087
4088         if (dst[0] == 'h')
4089                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4090
4091         instr->type = INSTR_ALU_AND_I;
4092         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4093         instr->alu.dst.n_bits = fdst->n_bits;
4094         instr->alu.dst.offset = fdst->offset / 8;
4095         instr->alu.src_val = src_val;
4096         return 0;
4097 }
4098
4099 static int
4100 instr_alu_or_translate(struct rte_swx_pipeline *p,
4101                        struct action *action,
4102                        char **tokens,
4103                        int n_tokens,
4104                        struct instruction *instr,
4105                        struct instruction_data *data __rte_unused)
4106 {
4107         char *dst = tokens[1], *src = tokens[2];
4108         struct field *fdst, *fsrc;
4109         uint64_t src_val;
4110         uint32_t dst_struct_id, src_struct_id;
4111
4112         CHECK(n_tokens == 3, EINVAL);
4113
4114         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4115         CHECK(fdst, EINVAL);
4116
4117         /* OR or OR_S. */
4118         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4119         if (fsrc) {
4120                 instr->type = INSTR_ALU_OR;
4121                 if ((dst[0] == 'h' && src[0] != 'h') ||
4122                     (dst[0] != 'h' && src[0] == 'h'))
4123                         instr->type = INSTR_ALU_OR_S;
4124
4125                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4126                 instr->alu.dst.n_bits = fdst->n_bits;
4127                 instr->alu.dst.offset = fdst->offset / 8;
4128                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4129                 instr->alu.src.n_bits = fsrc->n_bits;
4130                 instr->alu.src.offset = fsrc->offset / 8;
4131                 return 0;
4132         }
4133
4134         /* OR_I. */
4135         src_val = strtoull(src, &src, 0);
4136         CHECK(!src[0], EINVAL);
4137
4138         if (dst[0] == 'h')
4139                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4140
4141         instr->type = INSTR_ALU_OR_I;
4142         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4143         instr->alu.dst.n_bits = fdst->n_bits;
4144         instr->alu.dst.offset = fdst->offset / 8;
4145         instr->alu.src_val = src_val;
4146         return 0;
4147 }
4148
4149 static int
4150 instr_alu_xor_translate(struct rte_swx_pipeline *p,
4151                         struct action *action,
4152                         char **tokens,
4153                         int n_tokens,
4154                         struct instruction *instr,
4155                         struct instruction_data *data __rte_unused)
4156 {
4157         char *dst = tokens[1], *src = tokens[2];
4158         struct field *fdst, *fsrc;
4159         uint64_t src_val;
4160         uint32_t dst_struct_id, src_struct_id;
4161
4162         CHECK(n_tokens == 3, EINVAL);
4163
4164         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4165         CHECK(fdst, EINVAL);
4166
4167         /* XOR or XOR_S. */
4168         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4169         if (fsrc) {
4170                 instr->type = INSTR_ALU_XOR;
4171                 if ((dst[0] == 'h' && src[0] != 'h') ||
4172                     (dst[0] != 'h' && src[0] == 'h'))
4173                         instr->type = INSTR_ALU_XOR_S;
4174
4175                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4176                 instr->alu.dst.n_bits = fdst->n_bits;
4177                 instr->alu.dst.offset = fdst->offset / 8;
4178                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4179                 instr->alu.src.n_bits = fsrc->n_bits;
4180                 instr->alu.src.offset = fsrc->offset / 8;
4181                 return 0;
4182         }
4183
4184         /* XOR_I. */
4185         src_val = strtoull(src, &src, 0);
4186         CHECK(!src[0], EINVAL);
4187
4188         if (dst[0] == 'h')
4189                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4190
4191         instr->type = INSTR_ALU_XOR_I;
4192         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4193         instr->alu.dst.n_bits = fdst->n_bits;
4194         instr->alu.dst.offset = fdst->offset / 8;
4195         instr->alu.src_val = src_val;
4196         return 0;
4197 }
4198
4199 static inline void
4200 instr_alu_add_exec(struct rte_swx_pipeline *p)
4201 {
4202         struct thread *t = &p->threads[p->thread_id];
4203         struct instruction *ip = t->ip;
4204
4205         TRACE("[Thread %2u] add\n", p->thread_id);
4206
4207         /* Structs. */
4208         ALU(t, ip, +);
4209
4210         /* Thread. */
4211         thread_ip_inc(p);
4212 }
4213
4214 static inline void
4215 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
4216 {
4217         struct thread *t = &p->threads[p->thread_id];
4218         struct instruction *ip = t->ip;
4219
4220         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
4221
4222         /* Structs. */
4223         ALU_MH(t, ip, +);
4224
4225         /* Thread. */
4226         thread_ip_inc(p);
4227 }
4228
4229 static inline void
4230 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
4231 {
4232         struct thread *t = &p->threads[p->thread_id];
4233         struct instruction *ip = t->ip;
4234
4235         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
4236
4237         /* Structs. */
4238         ALU_HM(t, ip, +);
4239
4240         /* Thread. */
4241         thread_ip_inc(p);
4242 }
4243
4244 static inline void
4245 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
4246 {
4247         struct thread *t = &p->threads[p->thread_id];
4248         struct instruction *ip = t->ip;
4249
4250         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
4251
4252         /* Structs. */
4253         ALU_HH(t, ip, +);
4254
4255         /* Thread. */
4256         thread_ip_inc(p);
4257 }
4258
4259 static inline void
4260 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
4261 {
4262         struct thread *t = &p->threads[p->thread_id];
4263         struct instruction *ip = t->ip;
4264
4265         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
4266
4267         /* Structs. */
4268         ALU_MI(t, ip, +);
4269
4270         /* Thread. */
4271         thread_ip_inc(p);
4272 }
4273
4274 static inline void
4275 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
4276 {
4277         struct thread *t = &p->threads[p->thread_id];
4278         struct instruction *ip = t->ip;
4279
4280         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
4281
4282         /* Structs. */
4283         ALU_HI(t, ip, +);
4284
4285         /* Thread. */
4286         thread_ip_inc(p);
4287 }
4288
4289 static inline void
4290 instr_alu_sub_exec(struct rte_swx_pipeline *p)
4291 {
4292         struct thread *t = &p->threads[p->thread_id];
4293         struct instruction *ip = t->ip;
4294
4295         TRACE("[Thread %2u] sub\n", p->thread_id);
4296
4297         /* Structs. */
4298         ALU(t, ip, -);
4299
4300         /* Thread. */
4301         thread_ip_inc(p);
4302 }
4303
4304 static inline void
4305 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
4306 {
4307         struct thread *t = &p->threads[p->thread_id];
4308         struct instruction *ip = t->ip;
4309
4310         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
4311
4312         /* Structs. */
4313         ALU_MH(t, ip, -);
4314
4315         /* Thread. */
4316         thread_ip_inc(p);
4317 }
4318
4319 static inline void
4320 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4321 {
4322         struct thread *t = &p->threads[p->thread_id];
4323         struct instruction *ip = t->ip;
4324
4325         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4326
4327         /* Structs. */
4328         ALU_HM(t, ip, -);
4329
4330         /* Thread. */
4331         thread_ip_inc(p);
4332 }
4333
4334 static inline void
4335 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4336 {
4337         struct thread *t = &p->threads[p->thread_id];
4338         struct instruction *ip = t->ip;
4339
4340         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4341
4342         /* Structs. */
4343         ALU_HH(t, ip, -);
4344
4345         /* Thread. */
4346         thread_ip_inc(p);
4347 }
4348
4349 static inline void
4350 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4351 {
4352         struct thread *t = &p->threads[p->thread_id];
4353         struct instruction *ip = t->ip;
4354
4355         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4356
4357         /* Structs. */
4358         ALU_MI(t, ip, -);
4359
4360         /* Thread. */
4361         thread_ip_inc(p);
4362 }
4363
4364 static inline void
4365 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4366 {
4367         struct thread *t = &p->threads[p->thread_id];
4368         struct instruction *ip = t->ip;
4369
4370         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4371
4372         /* Structs. */
4373         ALU_HI(t, ip, -);
4374
4375         /* Thread. */
4376         thread_ip_inc(p);
4377 }
4378
4379 static inline void
4380 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4381 {
4382         struct thread *t = &p->threads[p->thread_id];
4383         struct instruction *ip = t->ip;
4384
4385         TRACE("[Thread %2u] shl\n", p->thread_id);
4386
4387         /* Structs. */
4388         ALU(t, ip, <<);
4389
4390         /* Thread. */
4391         thread_ip_inc(p);
4392 }
4393
4394 static inline void
4395 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4396 {
4397         struct thread *t = &p->threads[p->thread_id];
4398         struct instruction *ip = t->ip;
4399
4400         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4401
4402         /* Structs. */
4403         ALU_MH(t, ip, <<);
4404
4405         /* Thread. */
4406         thread_ip_inc(p);
4407 }
4408
4409 static inline void
4410 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4411 {
4412         struct thread *t = &p->threads[p->thread_id];
4413         struct instruction *ip = t->ip;
4414
4415         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4416
4417         /* Structs. */
4418         ALU_HM(t, ip, <<);
4419
4420         /* Thread. */
4421         thread_ip_inc(p);
4422 }
4423
4424 static inline void
4425 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4426 {
4427         struct thread *t = &p->threads[p->thread_id];
4428         struct instruction *ip = t->ip;
4429
4430         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4431
4432         /* Structs. */
4433         ALU_HH(t, ip, <<);
4434
4435         /* Thread. */
4436         thread_ip_inc(p);
4437 }
4438
4439 static inline void
4440 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4441 {
4442         struct thread *t = &p->threads[p->thread_id];
4443         struct instruction *ip = t->ip;
4444
4445         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4446
4447         /* Structs. */
4448         ALU_MI(t, ip, <<);
4449
4450         /* Thread. */
4451         thread_ip_inc(p);
4452 }
4453
4454 static inline void
4455 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4456 {
4457         struct thread *t = &p->threads[p->thread_id];
4458         struct instruction *ip = t->ip;
4459
4460         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4461
4462         /* Structs. */
4463         ALU_HI(t, ip, <<);
4464
4465         /* Thread. */
4466         thread_ip_inc(p);
4467 }
4468
4469 static inline void
4470 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4471 {
4472         struct thread *t = &p->threads[p->thread_id];
4473         struct instruction *ip = t->ip;
4474
4475         TRACE("[Thread %2u] shr\n", p->thread_id);
4476
4477         /* Structs. */
4478         ALU(t, ip, >>);
4479
4480         /* Thread. */
4481         thread_ip_inc(p);
4482 }
4483
4484 static inline void
4485 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4486 {
4487         struct thread *t = &p->threads[p->thread_id];
4488         struct instruction *ip = t->ip;
4489
4490         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4491
4492         /* Structs. */
4493         ALU_MH(t, ip, >>);
4494
4495         /* Thread. */
4496         thread_ip_inc(p);
4497 }
4498
4499 static inline void
4500 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4501 {
4502         struct thread *t = &p->threads[p->thread_id];
4503         struct instruction *ip = t->ip;
4504
4505         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4506
4507         /* Structs. */
4508         ALU_HM(t, ip, >>);
4509
4510         /* Thread. */
4511         thread_ip_inc(p);
4512 }
4513
4514 static inline void
4515 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4516 {
4517         struct thread *t = &p->threads[p->thread_id];
4518         struct instruction *ip = t->ip;
4519
4520         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4521
4522         /* Structs. */
4523         ALU_HH(t, ip, >>);
4524
4525         /* Thread. */
4526         thread_ip_inc(p);
4527 }
4528
4529 static inline void
4530 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4531 {
4532         struct thread *t = &p->threads[p->thread_id];
4533         struct instruction *ip = t->ip;
4534
4535         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4536
4537         /* Structs. */
4538         ALU_MI(t, ip, >>);
4539
4540         /* Thread. */
4541         thread_ip_inc(p);
4542 }
4543
4544 static inline void
4545 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4546 {
4547         struct thread *t = &p->threads[p->thread_id];
4548         struct instruction *ip = t->ip;
4549
4550         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4551
4552         /* Structs. */
4553         ALU_HI(t, ip, >>);
4554
4555         /* Thread. */
4556         thread_ip_inc(p);
4557 }
4558
4559 static inline void
4560 instr_alu_and_exec(struct rte_swx_pipeline *p)
4561 {
4562         struct thread *t = &p->threads[p->thread_id];
4563         struct instruction *ip = t->ip;
4564
4565         TRACE("[Thread %2u] and\n", p->thread_id);
4566
4567         /* Structs. */
4568         ALU(t, ip, &);
4569
4570         /* Thread. */
4571         thread_ip_inc(p);
4572 }
4573
4574 static inline void
4575 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
4576 {
4577         struct thread *t = &p->threads[p->thread_id];
4578         struct instruction *ip = t->ip;
4579
4580         TRACE("[Thread %2u] and (s)\n", p->thread_id);
4581
4582         /* Structs. */
4583         ALU_S(t, ip, &);
4584
4585         /* Thread. */
4586         thread_ip_inc(p);
4587 }
4588
4589 static inline void
4590 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4591 {
4592         struct thread *t = &p->threads[p->thread_id];
4593         struct instruction *ip = t->ip;
4594
4595         TRACE("[Thread %2u] and (i)\n", p->thread_id);
4596
4597         /* Structs. */
4598         ALU_I(t, ip, &);
4599
4600         /* Thread. */
4601         thread_ip_inc(p);
4602 }
4603
4604 static inline void
4605 instr_alu_or_exec(struct rte_swx_pipeline *p)
4606 {
4607         struct thread *t = &p->threads[p->thread_id];
4608         struct instruction *ip = t->ip;
4609
4610         TRACE("[Thread %2u] or\n", p->thread_id);
4611
4612         /* Structs. */
4613         ALU(t, ip, |);
4614
4615         /* Thread. */
4616         thread_ip_inc(p);
4617 }
4618
4619 static inline void
4620 instr_alu_or_s_exec(struct rte_swx_pipeline *p)
4621 {
4622         struct thread *t = &p->threads[p->thread_id];
4623         struct instruction *ip = t->ip;
4624
4625         TRACE("[Thread %2u] or (s)\n", p->thread_id);
4626
4627         /* Structs. */
4628         ALU_S(t, ip, |);
4629
4630         /* Thread. */
4631         thread_ip_inc(p);
4632 }
4633
4634 static inline void
4635 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4636 {
4637         struct thread *t = &p->threads[p->thread_id];
4638         struct instruction *ip = t->ip;
4639
4640         TRACE("[Thread %2u] or (i)\n", p->thread_id);
4641
4642         /* Structs. */
4643         ALU_I(t, ip, |);
4644
4645         /* Thread. */
4646         thread_ip_inc(p);
4647 }
4648
4649 static inline void
4650 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4651 {
4652         struct thread *t = &p->threads[p->thread_id];
4653         struct instruction *ip = t->ip;
4654
4655         TRACE("[Thread %2u] xor\n", p->thread_id);
4656
4657         /* Structs. */
4658         ALU(t, ip, ^);
4659
4660         /* Thread. */
4661         thread_ip_inc(p);
4662 }
4663
4664 static inline void
4665 instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
4666 {
4667         struct thread *t = &p->threads[p->thread_id];
4668         struct instruction *ip = t->ip;
4669
4670         TRACE("[Thread %2u] xor (s)\n", p->thread_id);
4671
4672         /* Structs. */
4673         ALU_S(t, ip, ^);
4674
4675         /* Thread. */
4676         thread_ip_inc(p);
4677 }
4678
4679 static inline void
4680 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4681 {
4682         struct thread *t = &p->threads[p->thread_id];
4683         struct instruction *ip = t->ip;
4684
4685         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
4686
4687         /* Structs. */
4688         ALU_I(t, ip, ^);
4689
4690         /* Thread. */
4691         thread_ip_inc(p);
4692 }
4693
4694 static inline void
4695 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4696 {
4697         struct thread *t = &p->threads[p->thread_id];
4698         struct instruction *ip = t->ip;
4699         uint8_t *dst_struct, *src_struct;
4700         uint16_t *dst16_ptr, dst;
4701         uint64_t *src64_ptr, src64, src64_mask, src;
4702         uint64_t r;
4703
4704         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
4705
4706         /* Structs. */
4707         dst_struct = t->structs[ip->alu.dst.struct_id];
4708         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4709         dst = *dst16_ptr;
4710
4711         src_struct = t->structs[ip->alu.src.struct_id];
4712         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4713         src64 = *src64_ptr;
4714         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4715         src = src64 & src64_mask;
4716
4717         r = dst;
4718         r = ~r & 0xFFFF;
4719
4720         /* The first input (r) is a 16-bit number. The second and the third
4721          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
4722          * three numbers (output r) is a 34-bit number.
4723          */
4724         r += (src >> 32) + (src & 0xFFFFFFFF);
4725
4726         /* The first input is a 16-bit number. The second input is an 18-bit
4727          * number. In the worst case scenario, the sum of the two numbers is a
4728          * 19-bit number.
4729          */
4730         r = (r & 0xFFFF) + (r >> 16);
4731
4732         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4733          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
4734          */
4735         r = (r & 0xFFFF) + (r >> 16);
4736
4737         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4738          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4739          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
4740          * therefore the output r is always a 16-bit number.
4741          */
4742         r = (r & 0xFFFF) + (r >> 16);
4743
4744         r = ~r & 0xFFFF;
4745         r = r ? r : 0xFFFF;
4746
4747         *dst16_ptr = (uint16_t)r;
4748
4749         /* Thread. */
4750         thread_ip_inc(p);
4751 }
4752
4753 static inline void
4754 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4755 {
4756         struct thread *t = &p->threads[p->thread_id];
4757         struct instruction *ip = t->ip;
4758         uint8_t *dst_struct, *src_struct;
4759         uint16_t *dst16_ptr, dst;
4760         uint64_t *src64_ptr, src64, src64_mask, src;
4761         uint64_t r;
4762
4763         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
4764
4765         /* Structs. */
4766         dst_struct = t->structs[ip->alu.dst.struct_id];
4767         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4768         dst = *dst16_ptr;
4769
4770         src_struct = t->structs[ip->alu.src.struct_id];
4771         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4772         src64 = *src64_ptr;
4773         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4774         src = src64 & src64_mask;
4775
4776         r = dst;
4777         r = ~r & 0xFFFF;
4778
4779         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
4780          * the following sequence of operations in 2's complement arithmetic:
4781          *    a '- b = (a - b) % 0xFFFF.
4782          *
4783          * In order to prevent an underflow for the below subtraction, in which
4784          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
4785          * minuend), we first add a multiple of the 0xFFFF modulus to the
4786          * minuend. The number we add to the minuend needs to be a 34-bit number
4787          * or higher, so for readability reasons we picked the 36-bit multiple.
4788          * We are effectively turning the 16-bit minuend into a 36-bit number:
4789          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
4790          */
4791         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
4792
4793         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
4794          * result (the output r) is a 36-bit number.
4795          */
4796         r -= (src >> 32) + (src & 0xFFFFFFFF);
4797
4798         /* The first input is a 16-bit number. The second input is a 20-bit
4799          * number. Their sum is a 21-bit number.
4800          */
4801         r = (r & 0xFFFF) + (r >> 16);
4802
4803         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4804          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
4805          */
4806         r = (r & 0xFFFF) + (r >> 16);
4807
4808         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4809          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4810          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4811          * generated, therefore the output r is always a 16-bit number.
4812          */
4813         r = (r & 0xFFFF) + (r >> 16);
4814
4815         r = ~r & 0xFFFF;
4816         r = r ? r : 0xFFFF;
4817
4818         *dst16_ptr = (uint16_t)r;
4819
4820         /* Thread. */
4821         thread_ip_inc(p);
4822 }
4823
4824 static inline void
4825 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4826 {
4827         struct thread *t = &p->threads[p->thread_id];
4828         struct instruction *ip = t->ip;
4829         uint8_t *dst_struct, *src_struct;
4830         uint16_t *dst16_ptr;
4831         uint32_t *src32_ptr;
4832         uint64_t r0, r1;
4833
4834         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
4835
4836         /* Structs. */
4837         dst_struct = t->structs[ip->alu.dst.struct_id];
4838         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4839
4840         src_struct = t->structs[ip->alu.src.struct_id];
4841         src32_ptr = (uint32_t *)&src_struct[0];
4842
4843         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
4844         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
4845         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
4846         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
4847         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
4848
4849         /* The first input is a 16-bit number. The second input is a 19-bit
4850          * number. Their sum is a 20-bit number.
4851          */
4852         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4853
4854         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4855          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
4856          */
4857         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4858
4859         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4860          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4861          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
4862          * generated, therefore the output r is always a 16-bit number.
4863          */
4864         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4865
4866         r0 = ~r0 & 0xFFFF;
4867         r0 = r0 ? r0 : 0xFFFF;
4868
4869         *dst16_ptr = (uint16_t)r0;
4870
4871         /* Thread. */
4872         thread_ip_inc(p);
4873 }
4874
4875 static inline void
4876 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4877 {
4878         struct thread *t = &p->threads[p->thread_id];
4879         struct instruction *ip = t->ip;
4880         uint8_t *dst_struct, *src_struct;
4881         uint16_t *dst16_ptr;
4882         uint32_t *src32_ptr;
4883         uint64_t r = 0;
4884         uint32_t i;
4885
4886         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
4887
4888         /* Structs. */
4889         dst_struct = t->structs[ip->alu.dst.struct_id];
4890         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4891
4892         src_struct = t->structs[ip->alu.src.struct_id];
4893         src32_ptr = (uint32_t *)&src_struct[0];
4894
4895         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
4896          * Therefore, in the worst case scenario, a 35-bit number is added to a
4897          * 16-bit number (the input r), so the output r is 36-bit number.
4898          */
4899         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
4900                 r += *src32_ptr;
4901
4902         /* The first input is a 16-bit number. The second input is a 20-bit
4903          * number. Their sum is a 21-bit number.
4904          */
4905         r = (r & 0xFFFF) + (r >> 16);
4906
4907         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4908          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
4909          */
4910         r = (r & 0xFFFF) + (r >> 16);
4911
4912         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4913          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4914          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4915          * generated, therefore the output r is always a 16-bit number.
4916          */
4917         r = (r & 0xFFFF) + (r >> 16);
4918
4919         r = ~r & 0xFFFF;
4920         r = r ? r : 0xFFFF;
4921
4922         *dst16_ptr = (uint16_t)r;
4923
4924         /* Thread. */
4925         thread_ip_inc(p);
4926 }
4927
4928 /*
4929  * Register array.
4930  */
4931 static struct regarray *
4932 regarray_find(struct rte_swx_pipeline *p, const char *name);
4933
4934 static int
4935 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4936                       struct action *action,
4937                       char **tokens,
4938                       int n_tokens,
4939                       struct instruction *instr,
4940                       struct instruction_data *data __rte_unused)
4941 {
4942         char *regarray = tokens[1], *idx = tokens[2];
4943         struct regarray *r;
4944         struct field *fidx;
4945         uint32_t idx_struct_id, idx_val;
4946
4947         CHECK(n_tokens == 3, EINVAL);
4948
4949         r = regarray_find(p, regarray);
4950         CHECK(r, EINVAL);
4951
4952         /* REGPREFETCH_RH, REGPREFETCH_RM. */
4953         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4954         if (fidx) {
4955                 instr->type = INSTR_REGPREFETCH_RM;
4956                 if (idx[0] == 'h')
4957                         instr->type = INSTR_REGPREFETCH_RH;
4958
4959                 instr->regarray.regarray_id = r->id;
4960                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4961                 instr->regarray.idx.n_bits = fidx->n_bits;
4962                 instr->regarray.idx.offset = fidx->offset / 8;
4963                 instr->regarray.dstsrc_val = 0; /* Unused. */
4964                 return 0;
4965         }
4966
4967         /* REGPREFETCH_RI. */
4968         idx_val = strtoul(idx, &idx, 0);
4969         CHECK(!idx[0], EINVAL);
4970
4971         instr->type = INSTR_REGPREFETCH_RI;
4972         instr->regarray.regarray_id = r->id;
4973         instr->regarray.idx_val = idx_val;
4974         instr->regarray.dstsrc_val = 0; /* Unused. */
4975         return 0;
4976 }
4977
4978 static int
4979 instr_regrd_translate(struct rte_swx_pipeline *p,
4980                       struct action *action,
4981                       char **tokens,
4982                       int n_tokens,
4983                       struct instruction *instr,
4984                       struct instruction_data *data __rte_unused)
4985 {
4986         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4987         struct regarray *r;
4988         struct field *fdst, *fidx;
4989         uint32_t dst_struct_id, idx_struct_id, idx_val;
4990
4991         CHECK(n_tokens == 4, EINVAL);
4992
4993         r = regarray_find(p, regarray);
4994         CHECK(r, EINVAL);
4995
4996         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4997         CHECK(fdst, EINVAL);
4998
4999         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
5000         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5001         if (fidx) {
5002                 instr->type = INSTR_REGRD_MRM;
5003                 if (dst[0] == 'h' && idx[0] != 'h')
5004                         instr->type = INSTR_REGRD_HRM;
5005                 if (dst[0] != 'h' && idx[0] == 'h')
5006                         instr->type = INSTR_REGRD_MRH;
5007                 if (dst[0] == 'h' && idx[0] == 'h')
5008                         instr->type = INSTR_REGRD_HRH;
5009
5010                 instr->regarray.regarray_id = r->id;
5011                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5012                 instr->regarray.idx.n_bits = fidx->n_bits;
5013                 instr->regarray.idx.offset = fidx->offset / 8;
5014                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5015                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
5016                 instr->regarray.dstsrc.offset = fdst->offset / 8;
5017                 return 0;
5018         }
5019
5020         /* REGRD_MRI, REGRD_HRI. */
5021         idx_val = strtoul(idx, &idx, 0);
5022         CHECK(!idx[0], EINVAL);
5023
5024         instr->type = INSTR_REGRD_MRI;
5025         if (dst[0] == 'h')
5026                 instr->type = INSTR_REGRD_HRI;
5027
5028         instr->regarray.regarray_id = r->id;
5029         instr->regarray.idx_val = idx_val;
5030         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5031         instr->regarray.dstsrc.n_bits = fdst->n_bits;
5032         instr->regarray.dstsrc.offset = fdst->offset / 8;
5033         return 0;
5034 }
5035
5036 static int
5037 instr_regwr_translate(struct rte_swx_pipeline *p,
5038                       struct action *action,
5039                       char **tokens,
5040                       int n_tokens,
5041                       struct instruction *instr,
5042                       struct instruction_data *data __rte_unused)
5043 {
5044         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5045         struct regarray *r;
5046         struct field *fidx, *fsrc;
5047         uint64_t src_val;
5048         uint32_t idx_struct_id, idx_val, src_struct_id;
5049
5050         CHECK(n_tokens == 4, EINVAL);
5051
5052         r = regarray_find(p, regarray);
5053         CHECK(r, EINVAL);
5054
5055         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
5056         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5057         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5058         if (fidx && fsrc) {
5059                 instr->type = INSTR_REGWR_RMM;
5060                 if (idx[0] == 'h' && src[0] != 'h')
5061                         instr->type = INSTR_REGWR_RHM;
5062                 if (idx[0] != 'h' && src[0] == 'h')
5063                         instr->type = INSTR_REGWR_RMH;
5064                 if (idx[0] == 'h' && src[0] == 'h')
5065                         instr->type = INSTR_REGWR_RHH;
5066
5067                 instr->regarray.regarray_id = r->id;
5068                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5069                 instr->regarray.idx.n_bits = fidx->n_bits;
5070                 instr->regarray.idx.offset = fidx->offset / 8;
5071                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5072                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5073                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5074                 return 0;
5075         }
5076
5077         /* REGWR_RHI, REGWR_RMI. */
5078         if (fidx && !fsrc) {
5079                 src_val = strtoull(src, &src, 0);
5080                 CHECK(!src[0], EINVAL);
5081
5082                 instr->type = INSTR_REGWR_RMI;
5083                 if (idx[0] == 'h')
5084                         instr->type = INSTR_REGWR_RHI;
5085
5086                 instr->regarray.regarray_id = r->id;
5087                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5088                 instr->regarray.idx.n_bits = fidx->n_bits;
5089                 instr->regarray.idx.offset = fidx->offset / 8;
5090                 instr->regarray.dstsrc_val = src_val;
5091                 return 0;
5092         }
5093
5094         /* REGWR_RIH, REGWR_RIM. */
5095         if (!fidx && fsrc) {
5096                 idx_val = strtoul(idx, &idx, 0);
5097                 CHECK(!idx[0], EINVAL);
5098
5099                 instr->type = INSTR_REGWR_RIM;
5100                 if (src[0] == 'h')
5101                         instr->type = INSTR_REGWR_RIH;
5102
5103                 instr->regarray.regarray_id = r->id;
5104                 instr->regarray.idx_val = idx_val;
5105                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5106                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5107                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5108                 return 0;
5109         }
5110
5111         /* REGWR_RII. */
5112         src_val = strtoull(src, &src, 0);
5113         CHECK(!src[0], EINVAL);
5114
5115         idx_val = strtoul(idx, &idx, 0);
5116         CHECK(!idx[0], EINVAL);
5117
5118         instr->type = INSTR_REGWR_RII;
5119         instr->regarray.idx_val = idx_val;
5120         instr->regarray.dstsrc_val = src_val;
5121
5122         return 0;
5123 }
5124
5125 static int
5126 instr_regadd_translate(struct rte_swx_pipeline *p,
5127                        struct action *action,
5128                        char **tokens,
5129                        int n_tokens,
5130                        struct instruction *instr,
5131                        struct instruction_data *data __rte_unused)
5132 {
5133         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5134         struct regarray *r;
5135         struct field *fidx, *fsrc;
5136         uint64_t src_val;
5137         uint32_t idx_struct_id, idx_val, src_struct_id;
5138
5139         CHECK(n_tokens == 4, EINVAL);
5140
5141         r = regarray_find(p, regarray);
5142         CHECK(r, EINVAL);
5143
5144         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
5145         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5146         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5147         if (fidx && fsrc) {
5148                 instr->type = INSTR_REGADD_RMM;
5149                 if (idx[0] == 'h' && src[0] != 'h')
5150                         instr->type = INSTR_REGADD_RHM;
5151                 if (idx[0] != 'h' && src[0] == 'h')
5152                         instr->type = INSTR_REGADD_RMH;
5153                 if (idx[0] == 'h' && src[0] == 'h')
5154                         instr->type = INSTR_REGADD_RHH;
5155
5156                 instr->regarray.regarray_id = r->id;
5157                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5158                 instr->regarray.idx.n_bits = fidx->n_bits;
5159                 instr->regarray.idx.offset = fidx->offset / 8;
5160                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5161                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5162                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5163                 return 0;
5164         }
5165
5166         /* REGADD_RHI, REGADD_RMI. */
5167         if (fidx && !fsrc) {
5168                 src_val = strtoull(src, &src, 0);
5169                 CHECK(!src[0], EINVAL);
5170
5171                 instr->type = INSTR_REGADD_RMI;
5172                 if (idx[0] == 'h')
5173                         instr->type = INSTR_REGADD_RHI;
5174
5175                 instr->regarray.regarray_id = r->id;
5176                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5177                 instr->regarray.idx.n_bits = fidx->n_bits;
5178                 instr->regarray.idx.offset = fidx->offset / 8;
5179                 instr->regarray.dstsrc_val = src_val;
5180                 return 0;
5181         }
5182
5183         /* REGADD_RIH, REGADD_RIM. */
5184         if (!fidx && fsrc) {
5185                 idx_val = strtoul(idx, &idx, 0);
5186                 CHECK(!idx[0], EINVAL);
5187
5188                 instr->type = INSTR_REGADD_RIM;
5189                 if (src[0] == 'h')
5190                         instr->type = INSTR_REGADD_RIH;
5191
5192                 instr->regarray.regarray_id = r->id;
5193                 instr->regarray.idx_val = idx_val;
5194                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5195                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5196                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5197                 return 0;
5198         }
5199
5200         /* REGADD_RII. */
5201         src_val = strtoull(src, &src, 0);
5202         CHECK(!src[0], EINVAL);
5203
5204         idx_val = strtoul(idx, &idx, 0);
5205         CHECK(!idx[0], EINVAL);
5206
5207         instr->type = INSTR_REGADD_RII;
5208         instr->regarray.idx_val = idx_val;
5209         instr->regarray.dstsrc_val = src_val;
5210         return 0;
5211 }
5212
5213 static inline uint64_t *
5214 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
5215 {
5216         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5217         return r->regarray;
5218 }
5219
5220 static inline uint64_t
5221 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5222 {
5223         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5224
5225         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5226         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5227         uint64_t idx64 = *idx64_ptr;
5228         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
5229         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5230
5231         return idx;
5232 }
5233
5234 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5235
5236 static inline uint64_t
5237 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5238 {
5239         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5240
5241         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5242         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5243         uint64_t idx64 = *idx64_ptr;
5244         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
5245
5246         return idx;
5247 }
5248
5249 #else
5250
5251 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
5252
5253 #endif
5254
5255 static inline uint64_t
5256 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5257 {
5258         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5259
5260         uint64_t idx = ip->regarray.idx_val & r->size_mask;
5261
5262         return idx;
5263 }
5264
5265 static inline uint64_t
5266 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
5267 {
5268         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5269         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5270         uint64_t src64 = *src64_ptr;
5271         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5272         uint64_t src = src64 & src64_mask;
5273
5274         return src;
5275 }
5276
5277 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5278
5279 static inline uint64_t
5280 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
5281 {
5282         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5283         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5284         uint64_t src64 = *src64_ptr;
5285         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
5286
5287         return src;
5288 }
5289
5290 #else
5291
5292 #define instr_regarray_src_nbo instr_regarray_src_hbo
5293
5294 #endif
5295
5296 static inline void
5297 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5298 {
5299         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5300         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5301         uint64_t dst64 = *dst64_ptr;
5302         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5303
5304         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5305
5306 }
5307
5308 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5309
5310 static inline void
5311 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5312 {
5313         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5314         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5315         uint64_t dst64 = *dst64_ptr;
5316         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5317
5318         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
5319         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5320 }
5321
5322 #else
5323
5324 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
5325
5326 #endif
5327
5328 static inline void
5329 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
5330 {
5331         struct thread *t = &p->threads[p->thread_id];
5332         struct instruction *ip = t->ip;
5333         uint64_t *regarray, idx;
5334
5335         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
5336
5337         /* Structs. */
5338         regarray = instr_regarray_regarray(p, ip);
5339         idx = instr_regarray_idx_nbo(p, t, ip);
5340         rte_prefetch0(&regarray[idx]);
5341
5342         /* Thread. */
5343         thread_ip_inc(p);
5344 }
5345
5346 static inline void
5347 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
5348 {
5349         struct thread *t = &p->threads[p->thread_id];
5350         struct instruction *ip = t->ip;
5351         uint64_t *regarray, idx;
5352
5353         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
5354
5355         /* Structs. */
5356         regarray = instr_regarray_regarray(p, ip);
5357         idx = instr_regarray_idx_hbo(p, t, ip);
5358         rte_prefetch0(&regarray[idx]);
5359
5360         /* Thread. */
5361         thread_ip_inc(p);
5362 }
5363
5364 static inline void
5365 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
5366 {
5367         struct thread *t = &p->threads[p->thread_id];
5368         struct instruction *ip = t->ip;
5369         uint64_t *regarray, idx;
5370
5371         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
5372
5373         /* Structs. */
5374         regarray = instr_regarray_regarray(p, ip);
5375         idx = instr_regarray_idx_imm(p, ip);
5376         rte_prefetch0(&regarray[idx]);
5377
5378         /* Thread. */
5379         thread_ip_inc(p);
5380 }
5381
5382 static inline void
5383 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
5384 {
5385         struct thread *t = &p->threads[p->thread_id];
5386         struct instruction *ip = t->ip;
5387         uint64_t *regarray, idx;
5388
5389         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
5390
5391         /* Structs. */
5392         regarray = instr_regarray_regarray(p, ip);
5393         idx = instr_regarray_idx_nbo(p, t, ip);
5394         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5395
5396         /* Thread. */
5397         thread_ip_inc(p);
5398 }
5399
5400 static inline void
5401 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
5402 {
5403         struct thread *t = &p->threads[p->thread_id];
5404         struct instruction *ip = t->ip;
5405         uint64_t *regarray, idx;
5406
5407         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
5408
5409         /* Structs. */
5410         regarray = instr_regarray_regarray(p, ip);
5411         idx = instr_regarray_idx_hbo(p, t, ip);
5412         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5413
5414         /* Thread. */
5415         thread_ip_inc(p);
5416 }
5417
5418 static inline void
5419 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
5420 {
5421         struct thread *t = &p->threads[p->thread_id];
5422         struct instruction *ip = t->ip;
5423         uint64_t *regarray, idx;
5424
5425         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
5426
5427         /* Structs. */
5428         regarray = instr_regarray_regarray(p, ip);
5429         idx = instr_regarray_idx_nbo(p, t, ip);
5430         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5431
5432         /* Thread. */
5433         thread_ip_inc(p);
5434 }
5435
5436 static inline void
5437 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
5438 {
5439         struct thread *t = &p->threads[p->thread_id];
5440         struct instruction *ip = t->ip;
5441         uint64_t *regarray, idx;
5442
5443         /* Structs. */
5444         regarray = instr_regarray_regarray(p, ip);
5445         idx = instr_regarray_idx_hbo(p, t, ip);
5446         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5447
5448         /* Thread. */
5449         thread_ip_inc(p);
5450 }
5451
5452 static inline void
5453 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
5454 {
5455         struct thread *t = &p->threads[p->thread_id];
5456         struct instruction *ip = t->ip;
5457         uint64_t *regarray, idx;
5458
5459         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
5460
5461         /* Structs. */
5462         regarray = instr_regarray_regarray(p, ip);
5463         idx = instr_regarray_idx_imm(p, ip);
5464         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5465
5466         /* Thread. */
5467         thread_ip_inc(p);
5468 }
5469
5470 static inline void
5471 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
5472 {
5473         struct thread *t = &p->threads[p->thread_id];
5474         struct instruction *ip = t->ip;
5475         uint64_t *regarray, idx;
5476
5477         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
5478
5479         /* Structs. */
5480         regarray = instr_regarray_regarray(p, ip);
5481         idx = instr_regarray_idx_imm(p, ip);
5482         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5483
5484         /* Thread. */
5485         thread_ip_inc(p);
5486 }
5487
5488 static inline void
5489 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
5490 {
5491         struct thread *t = &p->threads[p->thread_id];
5492         struct instruction *ip = t->ip;
5493         uint64_t *regarray, idx, src;
5494
5495         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
5496
5497         /* Structs. */
5498         regarray = instr_regarray_regarray(p, ip);
5499         idx = instr_regarray_idx_nbo(p, t, ip);
5500         src = instr_regarray_src_nbo(t, ip);
5501         regarray[idx] = src;
5502
5503         /* Thread. */
5504         thread_ip_inc(p);
5505 }
5506
5507 static inline void
5508 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
5509 {
5510         struct thread *t = &p->threads[p->thread_id];
5511         struct instruction *ip = t->ip;
5512         uint64_t *regarray, idx, src;
5513
5514         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
5515
5516         /* Structs. */
5517         regarray = instr_regarray_regarray(p, ip);
5518         idx = instr_regarray_idx_nbo(p, t, ip);
5519         src = instr_regarray_src_hbo(t, ip);
5520         regarray[idx] = src;
5521
5522         /* Thread. */
5523         thread_ip_inc(p);
5524 }
5525
5526 static inline void
5527 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
5528 {
5529         struct thread *t = &p->threads[p->thread_id];
5530         struct instruction *ip = t->ip;
5531         uint64_t *regarray, idx, src;
5532
5533         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
5534
5535         /* Structs. */
5536         regarray = instr_regarray_regarray(p, ip);
5537         idx = instr_regarray_idx_hbo(p, t, ip);
5538         src = instr_regarray_src_nbo(t, ip);
5539         regarray[idx] = src;
5540
5541         /* Thread. */
5542         thread_ip_inc(p);
5543 }
5544
5545 static inline void
5546 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
5547 {
5548         struct thread *t = &p->threads[p->thread_id];
5549         struct instruction *ip = t->ip;
5550         uint64_t *regarray, idx, src;
5551
5552         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
5553
5554         /* Structs. */
5555         regarray = instr_regarray_regarray(p, ip);
5556         idx = instr_regarray_idx_hbo(p, t, ip);
5557         src = instr_regarray_src_hbo(t, ip);
5558         regarray[idx] = src;
5559
5560         /* Thread. */
5561         thread_ip_inc(p);
5562 }
5563
5564 static inline void
5565 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
5566 {
5567         struct thread *t = &p->threads[p->thread_id];
5568         struct instruction *ip = t->ip;
5569         uint64_t *regarray, idx, src;
5570
5571         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
5572
5573         /* Structs. */
5574         regarray = instr_regarray_regarray(p, ip);
5575         idx = instr_regarray_idx_nbo(p, t, ip);
5576         src = ip->regarray.dstsrc_val;
5577         regarray[idx] = src;
5578
5579         /* Thread. */
5580         thread_ip_inc(p);
5581 }
5582
5583 static inline void
5584 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
5585 {
5586         struct thread *t = &p->threads[p->thread_id];
5587         struct instruction *ip = t->ip;
5588         uint64_t *regarray, idx, src;
5589
5590         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
5591
5592         /* Structs. */
5593         regarray = instr_regarray_regarray(p, ip);
5594         idx = instr_regarray_idx_hbo(p, t, ip);
5595         src = ip->regarray.dstsrc_val;
5596         regarray[idx] = src;
5597
5598         /* Thread. */
5599         thread_ip_inc(p);
5600 }
5601
5602 static inline void
5603 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
5604 {
5605         struct thread *t = &p->threads[p->thread_id];
5606         struct instruction *ip = t->ip;
5607         uint64_t *regarray, idx, src;
5608
5609         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
5610
5611         /* Structs. */
5612         regarray = instr_regarray_regarray(p, ip);
5613         idx = instr_regarray_idx_imm(p, ip);
5614         src = instr_regarray_src_nbo(t, ip);
5615         regarray[idx] = src;
5616
5617         /* Thread. */
5618         thread_ip_inc(p);
5619 }
5620
5621 static inline void
5622 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
5623 {
5624         struct thread *t = &p->threads[p->thread_id];
5625         struct instruction *ip = t->ip;
5626         uint64_t *regarray, idx, src;
5627
5628         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
5629
5630         /* Structs. */
5631         regarray = instr_regarray_regarray(p, ip);
5632         idx = instr_regarray_idx_imm(p, ip);
5633         src = instr_regarray_src_hbo(t, ip);
5634         regarray[idx] = src;
5635
5636         /* Thread. */
5637         thread_ip_inc(p);
5638 }
5639
5640 static inline void
5641 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
5642 {
5643         struct thread *t = &p->threads[p->thread_id];
5644         struct instruction *ip = t->ip;
5645         uint64_t *regarray, idx, src;
5646
5647         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
5648
5649         /* Structs. */
5650         regarray = instr_regarray_regarray(p, ip);
5651         idx = instr_regarray_idx_imm(p, ip);
5652         src = ip->regarray.dstsrc_val;
5653         regarray[idx] = src;
5654
5655         /* Thread. */
5656         thread_ip_inc(p);
5657 }
5658
5659 static inline void
5660 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
5661 {
5662         struct thread *t = &p->threads[p->thread_id];
5663         struct instruction *ip = t->ip;
5664         uint64_t *regarray, idx, src;
5665
5666         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
5667
5668         /* Structs. */
5669         regarray = instr_regarray_regarray(p, ip);
5670         idx = instr_regarray_idx_nbo(p, t, ip);
5671         src = instr_regarray_src_nbo(t, ip);
5672         regarray[idx] += src;
5673
5674         /* Thread. */
5675         thread_ip_inc(p);
5676 }
5677
5678 static inline void
5679 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
5680 {
5681         struct thread *t = &p->threads[p->thread_id];
5682         struct instruction *ip = t->ip;
5683         uint64_t *regarray, idx, src;
5684
5685         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
5686
5687         /* Structs. */
5688         regarray = instr_regarray_regarray(p, ip);
5689         idx = instr_regarray_idx_nbo(p, t, ip);
5690         src = instr_regarray_src_hbo(t, ip);
5691         regarray[idx] += src;
5692
5693         /* Thread. */
5694         thread_ip_inc(p);
5695 }
5696
5697 static inline void
5698 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
5699 {
5700         struct thread *t = &p->threads[p->thread_id];
5701         struct instruction *ip = t->ip;
5702         uint64_t *regarray, idx, src;
5703
5704         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
5705
5706         /* Structs. */
5707         regarray = instr_regarray_regarray(p, ip);
5708         idx = instr_regarray_idx_hbo(p, t, ip);
5709         src = instr_regarray_src_nbo(t, ip);
5710         regarray[idx] += src;
5711
5712         /* Thread. */
5713         thread_ip_inc(p);
5714 }
5715
5716 static inline void
5717 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
5718 {
5719         struct thread *t = &p->threads[p->thread_id];
5720         struct instruction *ip = t->ip;
5721         uint64_t *regarray, idx, src;
5722
5723         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
5724
5725         /* Structs. */
5726         regarray = instr_regarray_regarray(p, ip);
5727         idx = instr_regarray_idx_hbo(p, t, ip);
5728         src = instr_regarray_src_hbo(t, ip);
5729         regarray[idx] += src;
5730
5731         /* Thread. */
5732         thread_ip_inc(p);
5733 }
5734
5735 static inline void
5736 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
5737 {
5738         struct thread *t = &p->threads[p->thread_id];
5739         struct instruction *ip = t->ip;
5740         uint64_t *regarray, idx, src;
5741
5742         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
5743
5744         /* Structs. */
5745         regarray = instr_regarray_regarray(p, ip);
5746         idx = instr_regarray_idx_nbo(p, t, ip);
5747         src = ip->regarray.dstsrc_val;
5748         regarray[idx] += src;
5749
5750         /* Thread. */
5751         thread_ip_inc(p);
5752 }
5753
5754 static inline void
5755 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
5756 {
5757         struct thread *t = &p->threads[p->thread_id];
5758         struct instruction *ip = t->ip;
5759         uint64_t *regarray, idx, src;
5760
5761         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
5762
5763         /* Structs. */
5764         regarray = instr_regarray_regarray(p, ip);
5765         idx = instr_regarray_idx_hbo(p, t, ip);
5766         src = ip->regarray.dstsrc_val;
5767         regarray[idx] += src;
5768
5769         /* Thread. */
5770         thread_ip_inc(p);
5771 }
5772
5773 static inline void
5774 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
5775 {
5776         struct thread *t = &p->threads[p->thread_id];
5777         struct instruction *ip = t->ip;
5778         uint64_t *regarray, idx, src;
5779
5780         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
5781
5782         /* Structs. */
5783         regarray = instr_regarray_regarray(p, ip);
5784         idx = instr_regarray_idx_imm(p, ip);
5785         src = instr_regarray_src_nbo(t, ip);
5786         regarray[idx] += src;
5787
5788         /* Thread. */
5789         thread_ip_inc(p);
5790 }
5791
5792 static inline void
5793 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
5794 {
5795         struct thread *t = &p->threads[p->thread_id];
5796         struct instruction *ip = t->ip;
5797         uint64_t *regarray, idx, src;
5798
5799         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
5800
5801         /* Structs. */
5802         regarray = instr_regarray_regarray(p, ip);
5803         idx = instr_regarray_idx_imm(p, ip);
5804         src = instr_regarray_src_hbo(t, ip);
5805         regarray[idx] += src;
5806
5807         /* Thread. */
5808         thread_ip_inc(p);
5809 }
5810
5811 static inline void
5812 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
5813 {
5814         struct thread *t = &p->threads[p->thread_id];
5815         struct instruction *ip = t->ip;
5816         uint64_t *regarray, idx, src;
5817
5818         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
5819
5820         /* Structs. */
5821         regarray = instr_regarray_regarray(p, ip);
5822         idx = instr_regarray_idx_imm(p, ip);
5823         src = ip->regarray.dstsrc_val;
5824         regarray[idx] += src;
5825
5826         /* Thread. */
5827         thread_ip_inc(p);
5828 }
5829
5830 /*
5831  * metarray.
5832  */
5833 static struct metarray *
5834 metarray_find(struct rte_swx_pipeline *p, const char *name);
5835
5836 static int
5837 instr_metprefetch_translate(struct rte_swx_pipeline *p,
5838                             struct action *action,
5839                             char **tokens,
5840                             int n_tokens,
5841                             struct instruction *instr,
5842                             struct instruction_data *data __rte_unused)
5843 {
5844         char *metarray = tokens[1], *idx = tokens[2];
5845         struct metarray *m;
5846         struct field *fidx;
5847         uint32_t idx_struct_id, idx_val;
5848
5849         CHECK(n_tokens == 3, EINVAL);
5850
5851         m = metarray_find(p, metarray);
5852         CHECK(m, EINVAL);
5853
5854         /* METPREFETCH_H, METPREFETCH_M. */
5855         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5856         if (fidx) {
5857                 instr->type = INSTR_METPREFETCH_M;
5858                 if (idx[0] == 'h')
5859                         instr->type = INSTR_METPREFETCH_H;
5860
5861                 instr->meter.metarray_id = m->id;
5862                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5863                 instr->meter.idx.n_bits = fidx->n_bits;
5864                 instr->meter.idx.offset = fidx->offset / 8;
5865                 return 0;
5866         }
5867
5868         /* METPREFETCH_I. */
5869         idx_val = strtoul(idx, &idx, 0);
5870         CHECK(!idx[0], EINVAL);
5871
5872         instr->type = INSTR_METPREFETCH_I;
5873         instr->meter.metarray_id = m->id;
5874         instr->meter.idx_val = idx_val;
5875         return 0;
5876 }
5877
5878 static int
5879 instr_meter_translate(struct rte_swx_pipeline *p,
5880                       struct action *action,
5881                       char **tokens,
5882                       int n_tokens,
5883                       struct instruction *instr,
5884                       struct instruction_data *data __rte_unused)
5885 {
5886         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
5887         char *color_in = tokens[4], *color_out = tokens[5];
5888         struct metarray *m;
5889         struct field *fidx, *flength, *fcin, *fcout;
5890         uint32_t idx_struct_id, length_struct_id;
5891         uint32_t color_in_struct_id, color_out_struct_id;
5892
5893         CHECK(n_tokens == 6, EINVAL);
5894
5895         m = metarray_find(p, metarray);
5896         CHECK(m, EINVAL);
5897
5898         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5899
5900         flength = struct_field_parse(p, action, length, &length_struct_id);
5901         CHECK(flength, EINVAL);
5902
5903         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
5904
5905         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
5906         CHECK(fcout, EINVAL);
5907
5908         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
5909         if (fidx && fcin) {
5910                 instr->type = INSTR_METER_MMM;
5911                 if (idx[0] == 'h' && length[0] == 'h')
5912                         instr->type = INSTR_METER_HHM;
5913                 if (idx[0] == 'h' && length[0] != 'h')
5914                         instr->type = INSTR_METER_HMM;
5915                 if (idx[0] != 'h' && length[0] == 'h')
5916                         instr->type = INSTR_METER_MHM;
5917
5918                 instr->meter.metarray_id = m->id;
5919
5920                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5921                 instr->meter.idx.n_bits = fidx->n_bits;
5922                 instr->meter.idx.offset = fidx->offset / 8;
5923
5924                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5925                 instr->meter.length.n_bits = flength->n_bits;
5926                 instr->meter.length.offset = flength->offset / 8;
5927
5928                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5929                 instr->meter.color_in.n_bits = fcin->n_bits;
5930                 instr->meter.color_in.offset = fcin->offset / 8;
5931
5932                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5933                 instr->meter.color_out.n_bits = fcout->n_bits;
5934                 instr->meter.color_out.offset = fcout->offset / 8;
5935
5936                 return 0;
5937         }
5938
5939         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
5940         if (fidx && !fcin) {
5941                 uint32_t color_in_val = strtoul(color_in, &color_in, 0);
5942                 CHECK(!color_in[0], EINVAL);
5943
5944                 instr->type = INSTR_METER_MMI;
5945                 if (idx[0] == 'h' && length[0] == 'h')
5946                         instr->type = INSTR_METER_HHI;
5947                 if (idx[0] == 'h' && length[0] != 'h')
5948                         instr->type = INSTR_METER_HMI;
5949                 if (idx[0] != 'h' && length[0] == 'h')
5950                         instr->type = INSTR_METER_MHI;
5951
5952                 instr->meter.metarray_id = m->id;
5953
5954                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5955                 instr->meter.idx.n_bits = fidx->n_bits;
5956                 instr->meter.idx.offset = fidx->offset / 8;
5957
5958                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5959                 instr->meter.length.n_bits = flength->n_bits;
5960                 instr->meter.length.offset = flength->offset / 8;
5961
5962                 instr->meter.color_in_val = color_in_val;
5963
5964                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5965                 instr->meter.color_out.n_bits = fcout->n_bits;
5966                 instr->meter.color_out.offset = fcout->offset / 8;
5967
5968                 return 0;
5969         }
5970
5971         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5972         if (!fidx && fcin) {
5973                 uint32_t idx_val;
5974
5975                 idx_val = strtoul(idx, &idx, 0);
5976                 CHECK(!idx[0], EINVAL);
5977
5978                 instr->type = INSTR_METER_IMM;
5979                 if (length[0] == 'h')
5980                         instr->type = INSTR_METER_IHM;
5981
5982                 instr->meter.metarray_id = m->id;
5983
5984                 instr->meter.idx_val = idx_val;
5985
5986                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5987                 instr->meter.length.n_bits = flength->n_bits;
5988                 instr->meter.length.offset = flength->offset / 8;
5989
5990                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5991                 instr->meter.color_in.n_bits = fcin->n_bits;
5992                 instr->meter.color_in.offset = fcin->offset / 8;
5993
5994                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5995                 instr->meter.color_out.n_bits = fcout->n_bits;
5996                 instr->meter.color_out.offset = fcout->offset / 8;
5997
5998                 return 0;
5999         }
6000
6001         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
6002         if (!fidx && !fcin) {
6003                 uint32_t idx_val, color_in_val;
6004
6005                 idx_val = strtoul(idx, &idx, 0);
6006                 CHECK(!idx[0], EINVAL);
6007
6008                 color_in_val = strtoul(color_in, &color_in, 0);
6009                 CHECK(!color_in[0], EINVAL);
6010
6011                 instr->type = INSTR_METER_IMI;
6012                 if (length[0] == 'h')
6013                         instr->type = INSTR_METER_IHI;
6014
6015                 instr->meter.metarray_id = m->id;
6016
6017                 instr->meter.idx_val = idx_val;
6018
6019                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6020                 instr->meter.length.n_bits = flength->n_bits;
6021                 instr->meter.length.offset = flength->offset / 8;
6022
6023                 instr->meter.color_in_val = color_in_val;
6024
6025                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6026                 instr->meter.color_out.n_bits = fcout->n_bits;
6027                 instr->meter.color_out.offset = fcout->offset / 8;
6028
6029                 return 0;
6030         }
6031
6032         CHECK(0, EINVAL);
6033 }
6034
6035 static inline struct meter *
6036 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6037 {
6038         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6039
6040         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6041         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6042         uint64_t idx64 = *idx64_ptr;
6043         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
6044         uint64_t idx = idx64 & idx64_mask & r->size_mask;
6045
6046         return &r->metarray[idx];
6047 }
6048
6049 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6050
6051 static inline struct meter *
6052 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6053 {
6054         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6055
6056         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6057         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6058         uint64_t idx64 = *idx64_ptr;
6059         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
6060
6061         return &r->metarray[idx];
6062 }
6063
6064 #else
6065
6066 #define instr_meter_idx_nbo instr_meter_idx_hbo
6067
6068 #endif
6069
6070 static inline struct meter *
6071 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
6072 {
6073         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6074
6075         uint64_t idx =  ip->meter.idx_val & r->size_mask;
6076
6077         return &r->metarray[idx];
6078 }
6079
6080 static inline uint32_t
6081 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
6082 {
6083         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6084         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6085         uint64_t src64 = *src64_ptr;
6086         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
6087         uint64_t src = src64 & src64_mask;
6088
6089         return (uint32_t)src;
6090 }
6091
6092 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6093
6094 static inline uint32_t
6095 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
6096 {
6097         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6098         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6099         uint64_t src64 = *src64_ptr;
6100         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
6101
6102         return (uint32_t)src;
6103 }
6104
6105 #else
6106
6107 #define instr_meter_length_nbo instr_meter_length_hbo
6108
6109 #endif
6110
6111 static inline enum rte_color
6112 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
6113 {
6114         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
6115         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
6116         uint64_t src64 = *src64_ptr;
6117         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
6118         uint64_t src = src64 & src64_mask;
6119
6120         return (enum rte_color)src;
6121 }
6122
6123 static inline void
6124 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
6125 {
6126         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
6127         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
6128         uint64_t dst64 = *dst64_ptr;
6129         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
6130
6131         uint64_t src = (uint64_t)color_out;
6132
6133         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6134 }
6135
6136 static inline void
6137 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
6138 {
6139         struct thread *t = &p->threads[p->thread_id];
6140         struct instruction *ip = t->ip;
6141         struct meter *m;
6142
6143         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
6144
6145         /* Structs. */
6146         m = instr_meter_idx_nbo(p, t, ip);
6147         rte_prefetch0(m);
6148
6149         /* Thread. */
6150         thread_ip_inc(p);
6151 }
6152
6153 static inline void
6154 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
6155 {
6156         struct thread *t = &p->threads[p->thread_id];
6157         struct instruction *ip = t->ip;
6158         struct meter *m;
6159
6160         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
6161
6162         /* Structs. */
6163         m = instr_meter_idx_hbo(p, t, ip);
6164         rte_prefetch0(m);
6165
6166         /* Thread. */
6167         thread_ip_inc(p);
6168 }
6169
6170 static inline void
6171 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
6172 {
6173         struct thread *t = &p->threads[p->thread_id];
6174         struct instruction *ip = t->ip;
6175         struct meter *m;
6176
6177         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
6178
6179         /* Structs. */
6180         m = instr_meter_idx_imm(p, ip);
6181         rte_prefetch0(m);
6182
6183         /* Thread. */
6184         thread_ip_inc(p);
6185 }
6186
6187 static inline void
6188 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
6189 {
6190         struct thread *t = &p->threads[p->thread_id];
6191         struct instruction *ip = t->ip;
6192         struct meter *m;
6193         uint64_t time, n_pkts, n_bytes;
6194         uint32_t length;
6195         enum rte_color color_in, color_out;
6196
6197         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
6198
6199         /* Structs. */
6200         m = instr_meter_idx_nbo(p, t, ip);
6201         rte_prefetch0(m->n_pkts);
6202         time = rte_get_tsc_cycles();
6203         length = instr_meter_length_nbo(t, ip);
6204         color_in = instr_meter_color_in_hbo(t, ip);
6205
6206         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6207                 &m->profile->profile,
6208                 time,
6209                 length,
6210                 color_in);
6211
6212         color_out &= m->color_mask;
6213
6214         n_pkts = m->n_pkts[color_out];
6215         n_bytes = m->n_bytes[color_out];
6216
6217         instr_meter_color_out_hbo_set(t, ip, color_out);
6218
6219         m->n_pkts[color_out] = n_pkts + 1;
6220         m->n_bytes[color_out] = n_bytes + length;
6221
6222         /* Thread. */
6223         thread_ip_inc(p);
6224 }
6225
6226 static inline void
6227 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
6228 {
6229         struct thread *t = &p->threads[p->thread_id];
6230         struct instruction *ip = t->ip;
6231         struct meter *m;
6232         uint64_t time, n_pkts, n_bytes;
6233         uint32_t length;
6234         enum rte_color color_in, color_out;
6235
6236         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
6237
6238         /* Structs. */
6239         m = instr_meter_idx_nbo(p, t, ip);
6240         rte_prefetch0(m->n_pkts);
6241         time = rte_get_tsc_cycles();
6242         length = instr_meter_length_nbo(t, ip);
6243         color_in = (enum rte_color)ip->meter.color_in_val;
6244
6245         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6246                 &m->profile->profile,
6247                 time,
6248                 length,
6249                 color_in);
6250
6251         color_out &= m->color_mask;
6252
6253         n_pkts = m->n_pkts[color_out];
6254         n_bytes = m->n_bytes[color_out];
6255
6256         instr_meter_color_out_hbo_set(t, ip, color_out);
6257
6258         m->n_pkts[color_out] = n_pkts + 1;
6259         m->n_bytes[color_out] = n_bytes + length;
6260
6261         /* Thread. */
6262         thread_ip_inc(p);
6263 }
6264
6265 static inline void
6266 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
6267 {
6268         struct thread *t = &p->threads[p->thread_id];
6269         struct instruction *ip = t->ip;
6270         struct meter *m;
6271         uint64_t time, n_pkts, n_bytes;
6272         uint32_t length;
6273         enum rte_color color_in, color_out;
6274
6275         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
6276
6277         /* Structs. */
6278         m = instr_meter_idx_nbo(p, t, ip);
6279         rte_prefetch0(m->n_pkts);
6280         time = rte_get_tsc_cycles();
6281         length = instr_meter_length_hbo(t, ip);
6282         color_in = instr_meter_color_in_hbo(t, ip);
6283
6284         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6285                 &m->profile->profile,
6286                 time,
6287                 length,
6288                 color_in);
6289
6290         color_out &= m->color_mask;
6291
6292         n_pkts = m->n_pkts[color_out];
6293         n_bytes = m->n_bytes[color_out];
6294
6295         instr_meter_color_out_hbo_set(t, ip, color_out);
6296
6297         m->n_pkts[color_out] = n_pkts + 1;
6298         m->n_bytes[color_out] = n_bytes + length;
6299
6300         /* Thread. */
6301         thread_ip_inc(p);
6302 }
6303 static inline void
6304 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
6305 {
6306         struct thread *t = &p->threads[p->thread_id];
6307         struct instruction *ip = t->ip;
6308         struct meter *m;
6309         uint64_t time, n_pkts, n_bytes;
6310         uint32_t length;
6311         enum rte_color color_in, color_out;
6312
6313         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
6314
6315         /* Structs. */
6316         m = instr_meter_idx_nbo(p, t, ip);
6317         rte_prefetch0(m->n_pkts);
6318         time = rte_get_tsc_cycles();
6319         length = instr_meter_length_hbo(t, ip);
6320         color_in = (enum rte_color)ip->meter.color_in_val;
6321
6322         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6323                 &m->profile->profile,
6324                 time,
6325                 length,
6326                 color_in);
6327
6328         color_out &= m->color_mask;
6329
6330         n_pkts = m->n_pkts[color_out];
6331         n_bytes = m->n_bytes[color_out];
6332
6333         instr_meter_color_out_hbo_set(t, ip, color_out);
6334
6335         m->n_pkts[color_out] = n_pkts + 1;
6336         m->n_bytes[color_out] = n_bytes + length;
6337
6338         /* Thread. */
6339         thread_ip_inc(p);
6340 }
6341
6342 static inline void
6343 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
6344 {
6345         struct thread *t = &p->threads[p->thread_id];
6346         struct instruction *ip = t->ip;
6347         struct meter *m;
6348         uint64_t time, n_pkts, n_bytes;
6349         uint32_t length;
6350         enum rte_color color_in, color_out;
6351
6352         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
6353
6354         /* Structs. */
6355         m = instr_meter_idx_hbo(p, t, ip);
6356         rte_prefetch0(m->n_pkts);
6357         time = rte_get_tsc_cycles();
6358         length = instr_meter_length_nbo(t, ip);
6359         color_in = instr_meter_color_in_hbo(t, ip);
6360
6361         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6362                 &m->profile->profile,
6363                 time,
6364                 length,
6365                 color_in);
6366
6367         color_out &= m->color_mask;
6368
6369         n_pkts = m->n_pkts[color_out];
6370         n_bytes = m->n_bytes[color_out];
6371
6372         instr_meter_color_out_hbo_set(t, ip, color_out);
6373
6374         m->n_pkts[color_out] = n_pkts + 1;
6375         m->n_bytes[color_out] = n_bytes + length;
6376
6377         /* Thread. */
6378         thread_ip_inc(p);
6379 }
6380
6381 static inline void
6382 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
6383 {
6384         struct thread *t = &p->threads[p->thread_id];
6385         struct instruction *ip = t->ip;
6386         struct meter *m;
6387         uint64_t time, n_pkts, n_bytes;
6388         uint32_t length;
6389         enum rte_color color_in, color_out;
6390
6391         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
6392
6393         /* Structs. */
6394         m = instr_meter_idx_hbo(p, t, ip);
6395         rte_prefetch0(m->n_pkts);
6396         time = rte_get_tsc_cycles();
6397         length = instr_meter_length_nbo(t, ip);
6398         color_in = (enum rte_color)ip->meter.color_in_val;
6399
6400         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6401                 &m->profile->profile,
6402                 time,
6403                 length,
6404                 color_in);
6405
6406         color_out &= m->color_mask;
6407
6408         n_pkts = m->n_pkts[color_out];
6409         n_bytes = m->n_bytes[color_out];
6410
6411         instr_meter_color_out_hbo_set(t, ip, color_out);
6412
6413         m->n_pkts[color_out] = n_pkts + 1;
6414         m->n_bytes[color_out] = n_bytes + length;
6415
6416         /* Thread. */
6417         thread_ip_inc(p);
6418 }
6419
6420 static inline void
6421 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
6422 {
6423         struct thread *t = &p->threads[p->thread_id];
6424         struct instruction *ip = t->ip;
6425         struct meter *m;
6426         uint64_t time, n_pkts, n_bytes;
6427         uint32_t length;
6428         enum rte_color color_in, color_out;
6429
6430         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
6431
6432         /* Structs. */
6433         m = instr_meter_idx_hbo(p, t, ip);
6434         rte_prefetch0(m->n_pkts);
6435         time = rte_get_tsc_cycles();
6436         length = instr_meter_length_hbo(t, ip);
6437         color_in = instr_meter_color_in_hbo(t, ip);
6438
6439         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6440                 &m->profile->profile,
6441                 time,
6442                 length,
6443                 color_in);
6444
6445         color_out &= m->color_mask;
6446
6447         n_pkts = m->n_pkts[color_out];
6448         n_bytes = m->n_bytes[color_out];
6449
6450         instr_meter_color_out_hbo_set(t, ip, color_out);
6451
6452         m->n_pkts[color_out] = n_pkts + 1;
6453         m->n_bytes[color_out] = n_bytes + length;
6454
6455         /* Thread. */
6456         thread_ip_inc(p);
6457 }
6458
6459 static inline void
6460 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
6461 {
6462         struct thread *t = &p->threads[p->thread_id];
6463         struct instruction *ip = t->ip;
6464         struct meter *m;
6465         uint64_t time, n_pkts, n_bytes;
6466         uint32_t length;
6467         enum rte_color color_in, color_out;
6468
6469         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
6470
6471         /* Structs. */
6472         m = instr_meter_idx_hbo(p, t, ip);
6473         rte_prefetch0(m->n_pkts);
6474         time = rte_get_tsc_cycles();
6475         length = instr_meter_length_hbo(t, ip);
6476         color_in = (enum rte_color)ip->meter.color_in_val;
6477
6478         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6479                 &m->profile->profile,
6480                 time,
6481                 length,
6482                 color_in);
6483
6484         color_out &= m->color_mask;
6485
6486         n_pkts = m->n_pkts[color_out];
6487         n_bytes = m->n_bytes[color_out];
6488
6489         instr_meter_color_out_hbo_set(t, ip, color_out);
6490
6491         m->n_pkts[color_out] = n_pkts + 1;
6492         m->n_bytes[color_out] = n_bytes + length;
6493
6494         /* Thread. */
6495         thread_ip_inc(p);
6496 }
6497
6498 static inline void
6499 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
6500 {
6501         struct thread *t = &p->threads[p->thread_id];
6502         struct instruction *ip = t->ip;
6503         struct meter *m;
6504         uint64_t time, n_pkts, n_bytes;
6505         uint32_t length;
6506         enum rte_color color_in, color_out;
6507
6508         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
6509
6510         /* Structs. */
6511         m = instr_meter_idx_imm(p, ip);
6512         rte_prefetch0(m->n_pkts);
6513         time = rte_get_tsc_cycles();
6514         length = instr_meter_length_nbo(t, ip);
6515         color_in = instr_meter_color_in_hbo(t, ip);
6516
6517         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6518                 &m->profile->profile,
6519                 time,
6520                 length,
6521                 color_in);
6522
6523         color_out &= m->color_mask;
6524
6525         n_pkts = m->n_pkts[color_out];
6526         n_bytes = m->n_bytes[color_out];
6527
6528         instr_meter_color_out_hbo_set(t, ip, color_out);
6529
6530         m->n_pkts[color_out] = n_pkts + 1;
6531         m->n_bytes[color_out] = n_bytes + length;
6532
6533         /* Thread. */
6534         thread_ip_inc(p);
6535 }
6536
6537 static inline void
6538 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
6539 {
6540         struct thread *t = &p->threads[p->thread_id];
6541         struct instruction *ip = t->ip;
6542         struct meter *m;
6543         uint64_t time, n_pkts, n_bytes;
6544         uint32_t length;
6545         enum rte_color color_in, color_out;
6546
6547         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
6548
6549         /* Structs. */
6550         m = instr_meter_idx_imm(p, ip);
6551         rte_prefetch0(m->n_pkts);
6552         time = rte_get_tsc_cycles();
6553         length = instr_meter_length_nbo(t, ip);
6554         color_in = (enum rte_color)ip->meter.color_in_val;
6555
6556         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6557                 &m->profile->profile,
6558                 time,
6559                 length,
6560                 color_in);
6561
6562         color_out &= m->color_mask;
6563
6564         n_pkts = m->n_pkts[color_out];
6565         n_bytes = m->n_bytes[color_out];
6566
6567         instr_meter_color_out_hbo_set(t, ip, color_out);
6568
6569         m->n_pkts[color_out] = n_pkts + 1;
6570         m->n_bytes[color_out] = n_bytes + length;
6571
6572         /* Thread. */
6573         thread_ip_inc(p);
6574 }
6575
6576 static inline void
6577 instr_meter_imm_exec(struct rte_swx_pipeline *p)
6578 {
6579         struct thread *t = &p->threads[p->thread_id];
6580         struct instruction *ip = t->ip;
6581         struct meter *m;
6582         uint64_t time, n_pkts, n_bytes;
6583         uint32_t length;
6584         enum rte_color color_in, color_out;
6585
6586         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
6587
6588         /* Structs. */
6589         m = instr_meter_idx_imm(p, ip);
6590         rte_prefetch0(m->n_pkts);
6591         time = rte_get_tsc_cycles();
6592         length = instr_meter_length_hbo(t, ip);
6593         color_in = instr_meter_color_in_hbo(t, ip);
6594
6595         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6596                 &m->profile->profile,
6597                 time,
6598                 length,
6599                 color_in);
6600
6601         color_out &= m->color_mask;
6602
6603         n_pkts = m->n_pkts[color_out];
6604         n_bytes = m->n_bytes[color_out];
6605
6606         instr_meter_color_out_hbo_set(t, ip, color_out);
6607
6608         m->n_pkts[color_out] = n_pkts + 1;
6609         m->n_bytes[color_out] = n_bytes + length;
6610
6611         /* Thread. */
6612         thread_ip_inc(p);
6613 }
6614 static inline void
6615 instr_meter_imi_exec(struct rte_swx_pipeline *p)
6616 {
6617         struct thread *t = &p->threads[p->thread_id];
6618         struct instruction *ip = t->ip;
6619         struct meter *m;
6620         uint64_t time, n_pkts, n_bytes;
6621         uint32_t length;
6622         enum rte_color color_in, color_out;
6623
6624         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
6625
6626         /* Structs. */
6627         m = instr_meter_idx_imm(p, ip);
6628         rte_prefetch0(m->n_pkts);
6629         time = rte_get_tsc_cycles();
6630         length = instr_meter_length_hbo(t, ip);
6631         color_in = (enum rte_color)ip->meter.color_in_val;
6632
6633         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6634                 &m->profile->profile,
6635                 time,
6636                 length,
6637                 color_in);
6638
6639         color_out &= m->color_mask;
6640
6641         n_pkts = m->n_pkts[color_out];
6642         n_bytes = m->n_bytes[color_out];
6643
6644         instr_meter_color_out_hbo_set(t, ip, color_out);
6645
6646         m->n_pkts[color_out] = n_pkts + 1;
6647         m->n_bytes[color_out] = n_bytes + length;
6648
6649         /* Thread. */
6650         thread_ip_inc(p);
6651 }
6652
6653 /*
6654  * jmp.
6655  */
6656 static struct action *
6657 action_find(struct rte_swx_pipeline *p, const char *name);
6658
6659 static int
6660 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
6661                     struct action *action __rte_unused,
6662                     char **tokens,
6663                     int n_tokens,
6664                     struct instruction *instr,
6665                     struct instruction_data *data)
6666 {
6667         CHECK(n_tokens == 2, EINVAL);
6668
6669         strcpy(data->jmp_label, tokens[1]);
6670
6671         instr->type = INSTR_JMP;
6672         instr->jmp.ip = NULL; /* Resolved later. */
6673         return 0;
6674 }
6675
6676 static int
6677 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
6678                           struct action *action __rte_unused,
6679                           char **tokens,
6680                           int n_tokens,
6681                           struct instruction *instr,
6682                           struct instruction_data *data)
6683 {
6684         struct header *h;
6685
6686         CHECK(n_tokens == 3, EINVAL);
6687
6688         strcpy(data->jmp_label, tokens[1]);
6689
6690         h = header_parse(p, tokens[2]);
6691         CHECK(h, EINVAL);
6692
6693         instr->type = INSTR_JMP_VALID;
6694         instr->jmp.ip = NULL; /* Resolved later. */
6695         instr->jmp.header_id = h->id;
6696         return 0;
6697 }
6698
6699 static int
6700 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
6701                             struct action *action __rte_unused,
6702                             char **tokens,
6703                             int n_tokens,
6704                             struct instruction *instr,
6705                             struct instruction_data *data)
6706 {
6707         struct header *h;
6708
6709         CHECK(n_tokens == 3, EINVAL);
6710
6711         strcpy(data->jmp_label, tokens[1]);
6712
6713         h = header_parse(p, tokens[2]);
6714         CHECK(h, EINVAL);
6715
6716         instr->type = INSTR_JMP_INVALID;
6717         instr->jmp.ip = NULL; /* Resolved later. */
6718         instr->jmp.header_id = h->id;
6719         return 0;
6720 }
6721
6722 static int
6723 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
6724                         struct action *action,
6725                         char **tokens,
6726                         int n_tokens,
6727                         struct instruction *instr,
6728                         struct instruction_data *data)
6729 {
6730         CHECK(!action, EINVAL);
6731         CHECK(n_tokens == 2, EINVAL);
6732
6733         strcpy(data->jmp_label, tokens[1]);
6734
6735         instr->type = INSTR_JMP_HIT;
6736         instr->jmp.ip = NULL; /* Resolved later. */
6737         return 0;
6738 }
6739
6740 static int
6741 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
6742                          struct action *action,
6743                          char **tokens,
6744                          int n_tokens,
6745                          struct instruction *instr,
6746                          struct instruction_data *data)
6747 {
6748         CHECK(!action, EINVAL);
6749         CHECK(n_tokens == 2, EINVAL);
6750
6751         strcpy(data->jmp_label, tokens[1]);
6752
6753         instr->type = INSTR_JMP_MISS;
6754         instr->jmp.ip = NULL; /* Resolved later. */
6755         return 0;
6756 }
6757
6758 static int
6759 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
6760                                struct action *action,
6761                                char **tokens,
6762                                int n_tokens,
6763                                struct instruction *instr,
6764                                struct instruction_data *data)
6765 {
6766         struct action *a;
6767
6768         CHECK(!action, EINVAL);
6769         CHECK(n_tokens == 3, EINVAL);
6770
6771         strcpy(data->jmp_label, tokens[1]);
6772
6773         a = action_find(p, tokens[2]);
6774         CHECK(a, EINVAL);
6775
6776         instr->type = INSTR_JMP_ACTION_HIT;
6777         instr->jmp.ip = NULL; /* Resolved later. */
6778         instr->jmp.action_id = a->id;
6779         return 0;
6780 }
6781
6782 static int
6783 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
6784                                 struct action *action,
6785                                 char **tokens,
6786                                 int n_tokens,
6787                                 struct instruction *instr,
6788                                 struct instruction_data *data)
6789 {
6790         struct action *a;
6791
6792         CHECK(!action, EINVAL);
6793         CHECK(n_tokens == 3, EINVAL);
6794
6795         strcpy(data->jmp_label, tokens[1]);
6796
6797         a = action_find(p, tokens[2]);
6798         CHECK(a, EINVAL);
6799
6800         instr->type = INSTR_JMP_ACTION_MISS;
6801         instr->jmp.ip = NULL; /* Resolved later. */
6802         instr->jmp.action_id = a->id;
6803         return 0;
6804 }
6805
6806 static int
6807 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
6808                        struct action *action,
6809                        char **tokens,
6810                        int n_tokens,
6811                        struct instruction *instr,
6812                        struct instruction_data *data)
6813 {
6814         char *a = tokens[2], *b = tokens[3];
6815         struct field *fa, *fb;
6816         uint64_t b_val;
6817         uint32_t a_struct_id, b_struct_id;
6818
6819         CHECK(n_tokens == 4, EINVAL);
6820
6821         strcpy(data->jmp_label, tokens[1]);
6822
6823         fa = struct_field_parse(p, action, a, &a_struct_id);
6824         CHECK(fa, EINVAL);
6825
6826         /* JMP_EQ or JMP_EQ_S. */
6827         fb = struct_field_parse(p, action, b, &b_struct_id);
6828         if (fb) {
6829                 instr->type = INSTR_JMP_EQ;
6830                 if ((a[0] == 'h' && b[0] != 'h') ||
6831                     (a[0] != 'h' && b[0] == 'h'))
6832                         instr->type = INSTR_JMP_EQ_S;
6833                 instr->jmp.ip = NULL; /* Resolved later. */
6834
6835                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6836                 instr->jmp.a.n_bits = fa->n_bits;
6837                 instr->jmp.a.offset = fa->offset / 8;
6838                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6839                 instr->jmp.b.n_bits = fb->n_bits;
6840                 instr->jmp.b.offset = fb->offset / 8;
6841                 return 0;
6842         }
6843
6844         /* JMP_EQ_I. */
6845         b_val = strtoull(b, &b, 0);
6846         CHECK(!b[0], EINVAL);
6847
6848         if (a[0] == 'h')
6849                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6850
6851         instr->type = INSTR_JMP_EQ_I;
6852         instr->jmp.ip = NULL; /* Resolved later. */
6853         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6854         instr->jmp.a.n_bits = fa->n_bits;
6855         instr->jmp.a.offset = fa->offset / 8;
6856         instr->jmp.b_val = b_val;
6857         return 0;
6858 }
6859
6860 static int
6861 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
6862                         struct action *action,
6863                         char **tokens,
6864                         int n_tokens,
6865                         struct instruction *instr,
6866                         struct instruction_data *data)
6867 {
6868         char *a = tokens[2], *b = tokens[3];
6869         struct field *fa, *fb;
6870         uint64_t b_val;
6871         uint32_t a_struct_id, b_struct_id;
6872
6873         CHECK(n_tokens == 4, EINVAL);
6874
6875         strcpy(data->jmp_label, tokens[1]);
6876
6877         fa = struct_field_parse(p, action, a, &a_struct_id);
6878         CHECK(fa, EINVAL);
6879
6880         /* JMP_NEQ or JMP_NEQ_S. */
6881         fb = struct_field_parse(p, action, b, &b_struct_id);
6882         if (fb) {
6883                 instr->type = INSTR_JMP_NEQ;
6884                 if ((a[0] == 'h' && b[0] != 'h') ||
6885                     (a[0] != 'h' && b[0] == 'h'))
6886                         instr->type = INSTR_JMP_NEQ_S;
6887                 instr->jmp.ip = NULL; /* Resolved later. */
6888
6889                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6890                 instr->jmp.a.n_bits = fa->n_bits;
6891                 instr->jmp.a.offset = fa->offset / 8;
6892                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6893                 instr->jmp.b.n_bits = fb->n_bits;
6894                 instr->jmp.b.offset = fb->offset / 8;
6895                 return 0;
6896         }
6897
6898         /* JMP_NEQ_I. */
6899         b_val = strtoull(b, &b, 0);
6900         CHECK(!b[0], EINVAL);
6901
6902         if (a[0] == 'h')
6903                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6904
6905         instr->type = INSTR_JMP_NEQ_I;
6906         instr->jmp.ip = NULL; /* Resolved later. */
6907         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6908         instr->jmp.a.n_bits = fa->n_bits;
6909         instr->jmp.a.offset = fa->offset / 8;
6910         instr->jmp.b_val = b_val;
6911         return 0;
6912 }
6913
6914 static int
6915 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
6916                        struct action *action,
6917                        char **tokens,
6918                        int n_tokens,
6919                        struct instruction *instr,
6920                        struct instruction_data *data)
6921 {
6922         char *a = tokens[2], *b = tokens[3];
6923         struct field *fa, *fb;
6924         uint64_t b_val;
6925         uint32_t a_struct_id, b_struct_id;
6926
6927         CHECK(n_tokens == 4, EINVAL);
6928
6929         strcpy(data->jmp_label, tokens[1]);
6930
6931         fa = struct_field_parse(p, action, a, &a_struct_id);
6932         CHECK(fa, EINVAL);
6933
6934         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
6935         fb = struct_field_parse(p, action, b, &b_struct_id);
6936         if (fb) {
6937                 instr->type = INSTR_JMP_LT;
6938                 if (a[0] == 'h' && b[0] != 'h')
6939                         instr->type = INSTR_JMP_LT_HM;
6940                 if (a[0] != 'h' && b[0] == 'h')
6941                         instr->type = INSTR_JMP_LT_MH;
6942                 if (a[0] == 'h' && b[0] == 'h')
6943                         instr->type = INSTR_JMP_LT_HH;
6944                 instr->jmp.ip = NULL; /* Resolved later. */
6945
6946                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6947                 instr->jmp.a.n_bits = fa->n_bits;
6948                 instr->jmp.a.offset = fa->offset / 8;
6949                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6950                 instr->jmp.b.n_bits = fb->n_bits;
6951                 instr->jmp.b.offset = fb->offset / 8;
6952                 return 0;
6953         }
6954
6955         /* JMP_LT_MI, JMP_LT_HI. */
6956         b_val = strtoull(b, &b, 0);
6957         CHECK(!b[0], EINVAL);
6958
6959         instr->type = INSTR_JMP_LT_MI;
6960         if (a[0] == 'h')
6961                 instr->type = INSTR_JMP_LT_HI;
6962         instr->jmp.ip = NULL; /* Resolved later. */
6963
6964         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6965         instr->jmp.a.n_bits = fa->n_bits;
6966         instr->jmp.a.offset = fa->offset / 8;
6967         instr->jmp.b_val = b_val;
6968         return 0;
6969 }
6970
6971 static int
6972 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
6973                        struct action *action,
6974                        char **tokens,
6975                        int n_tokens,
6976                        struct instruction *instr,
6977                        struct instruction_data *data)
6978 {
6979         char *a = tokens[2], *b = tokens[3];
6980         struct field *fa, *fb;
6981         uint64_t b_val;
6982         uint32_t a_struct_id, b_struct_id;
6983
6984         CHECK(n_tokens == 4, EINVAL);
6985
6986         strcpy(data->jmp_label, tokens[1]);
6987
6988         fa = struct_field_parse(p, action, a, &a_struct_id);
6989         CHECK(fa, EINVAL);
6990
6991         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
6992         fb = struct_field_parse(p, action, b, &b_struct_id);
6993         if (fb) {
6994                 instr->type = INSTR_JMP_GT;
6995                 if (a[0] == 'h' && b[0] != 'h')
6996                         instr->type = INSTR_JMP_GT_HM;
6997                 if (a[0] != 'h' && b[0] == 'h')
6998                         instr->type = INSTR_JMP_GT_MH;
6999                 if (a[0] == 'h' && b[0] == 'h')
7000                         instr->type = INSTR_JMP_GT_HH;
7001                 instr->jmp.ip = NULL; /* Resolved later. */
7002
7003                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7004                 instr->jmp.a.n_bits = fa->n_bits;
7005                 instr->jmp.a.offset = fa->offset / 8;
7006                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7007                 instr->jmp.b.n_bits = fb->n_bits;
7008                 instr->jmp.b.offset = fb->offset / 8;
7009                 return 0;
7010         }
7011
7012         /* JMP_GT_MI, JMP_GT_HI. */
7013         b_val = strtoull(b, &b, 0);
7014         CHECK(!b[0], EINVAL);
7015
7016         instr->type = INSTR_JMP_GT_MI;
7017         if (a[0] == 'h')
7018                 instr->type = INSTR_JMP_GT_HI;
7019         instr->jmp.ip = NULL; /* Resolved later. */
7020
7021         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7022         instr->jmp.a.n_bits = fa->n_bits;
7023         instr->jmp.a.offset = fa->offset / 8;
7024         instr->jmp.b_val = b_val;
7025         return 0;
7026 }
7027
7028 static inline void
7029 instr_jmp_exec(struct rte_swx_pipeline *p)
7030 {
7031         struct thread *t = &p->threads[p->thread_id];
7032         struct instruction *ip = t->ip;
7033
7034         TRACE("[Thread %2u] jmp\n", p->thread_id);
7035
7036         thread_ip_set(t, ip->jmp.ip);
7037 }
7038
7039 static inline void
7040 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
7041 {
7042         struct thread *t = &p->threads[p->thread_id];
7043         struct instruction *ip = t->ip;
7044         uint32_t header_id = ip->jmp.header_id;
7045
7046         TRACE("[Thread %2u] jmpv\n", p->thread_id);
7047
7048         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
7049 }
7050
7051 static inline void
7052 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
7053 {
7054         struct thread *t = &p->threads[p->thread_id];
7055         struct instruction *ip = t->ip;
7056         uint32_t header_id = ip->jmp.header_id;
7057
7058         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
7059
7060         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
7061 }
7062
7063 static inline void
7064 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
7065 {
7066         struct thread *t = &p->threads[p->thread_id];
7067         struct instruction *ip = t->ip;
7068         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
7069
7070         TRACE("[Thread %2u] jmph\n", p->thread_id);
7071
7072         t->ip = ip_next[t->hit];
7073 }
7074
7075 static inline void
7076 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
7077 {
7078         struct thread *t = &p->threads[p->thread_id];
7079         struct instruction *ip = t->ip;
7080         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
7081
7082         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
7083
7084         t->ip = ip_next[t->hit];
7085 }
7086
7087 static inline void
7088 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
7089 {
7090         struct thread *t = &p->threads[p->thread_id];
7091         struct instruction *ip = t->ip;
7092
7093         TRACE("[Thread %2u] jmpa\n", p->thread_id);
7094
7095         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
7096 }
7097
7098 static inline void
7099 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
7100 {
7101         struct thread *t = &p->threads[p->thread_id];
7102         struct instruction *ip = t->ip;
7103
7104         TRACE("[Thread %2u] jmpna\n", p->thread_id);
7105
7106         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
7107 }
7108
7109 static inline void
7110 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
7111 {
7112         struct thread *t = &p->threads[p->thread_id];
7113         struct instruction *ip = t->ip;
7114
7115         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
7116
7117         JMP_CMP(t, ip, ==);
7118 }
7119
7120 static inline void
7121 instr_jmp_eq_s_exec(struct rte_swx_pipeline *p)
7122 {
7123         struct thread *t = &p->threads[p->thread_id];
7124         struct instruction *ip = t->ip;
7125
7126         TRACE("[Thread %2u] jmpeq (s)\n", p->thread_id);
7127
7128         JMP_CMP_S(t, ip, ==);
7129 }
7130
7131 static inline void
7132 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
7133 {
7134         struct thread *t = &p->threads[p->thread_id];
7135         struct instruction *ip = t->ip;
7136
7137         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
7138
7139         JMP_CMP_I(t, ip, ==);
7140 }
7141
7142 static inline void
7143 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
7144 {
7145         struct thread *t = &p->threads[p->thread_id];
7146         struct instruction *ip = t->ip;
7147
7148         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
7149
7150         JMP_CMP(t, ip, !=);
7151 }
7152
7153 static inline void
7154 instr_jmp_neq_s_exec(struct rte_swx_pipeline *p)
7155 {
7156         struct thread *t = &p->threads[p->thread_id];
7157         struct instruction *ip = t->ip;
7158
7159         TRACE("[Thread %2u] jmpneq (s)\n", p->thread_id);
7160
7161         JMP_CMP_S(t, ip, !=);
7162 }
7163
7164 static inline void
7165 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
7166 {
7167         struct thread *t = &p->threads[p->thread_id];
7168         struct instruction *ip = t->ip;
7169
7170         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
7171
7172         JMP_CMP_I(t, ip, !=);
7173 }
7174
7175 static inline void
7176 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
7177 {
7178         struct thread *t = &p->threads[p->thread_id];
7179         struct instruction *ip = t->ip;
7180
7181         TRACE("[Thread %2u] jmplt\n", p->thread_id);
7182
7183         JMP_CMP(t, ip, <);
7184 }
7185
7186 static inline void
7187 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
7188 {
7189         struct thread *t = &p->threads[p->thread_id];
7190         struct instruction *ip = t->ip;
7191
7192         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
7193
7194         JMP_CMP_MH(t, ip, <);
7195 }
7196
7197 static inline void
7198 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
7199 {
7200         struct thread *t = &p->threads[p->thread_id];
7201         struct instruction *ip = t->ip;
7202
7203         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
7204
7205         JMP_CMP_HM(t, ip, <);
7206 }
7207
7208 static inline void
7209 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
7210 {
7211         struct thread *t = &p->threads[p->thread_id];
7212         struct instruction *ip = t->ip;
7213
7214         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
7215
7216         JMP_CMP_HH(t, ip, <);
7217 }
7218
7219 static inline void
7220 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
7221 {
7222         struct thread *t = &p->threads[p->thread_id];
7223         struct instruction *ip = t->ip;
7224
7225         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
7226
7227         JMP_CMP_MI(t, ip, <);
7228 }
7229
7230 static inline void
7231 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
7232 {
7233         struct thread *t = &p->threads[p->thread_id];
7234         struct instruction *ip = t->ip;
7235
7236         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
7237
7238         JMP_CMP_HI(t, ip, <);
7239 }
7240
7241 static inline void
7242 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
7243 {
7244         struct thread *t = &p->threads[p->thread_id];
7245         struct instruction *ip = t->ip;
7246
7247         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
7248
7249         JMP_CMP(t, ip, >);
7250 }
7251
7252 static inline void
7253 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
7254 {
7255         struct thread *t = &p->threads[p->thread_id];
7256         struct instruction *ip = t->ip;
7257
7258         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
7259
7260         JMP_CMP_MH(t, ip, >);
7261 }
7262
7263 static inline void
7264 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
7265 {
7266         struct thread *t = &p->threads[p->thread_id];
7267         struct instruction *ip = t->ip;
7268
7269         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
7270
7271         JMP_CMP_HM(t, ip, >);
7272 }
7273
7274 static inline void
7275 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
7276 {
7277         struct thread *t = &p->threads[p->thread_id];
7278         struct instruction *ip = t->ip;
7279
7280         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
7281
7282         JMP_CMP_HH(t, ip, >);
7283 }
7284
7285 static inline void
7286 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
7287 {
7288         struct thread *t = &p->threads[p->thread_id];
7289         struct instruction *ip = t->ip;
7290
7291         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
7292
7293         JMP_CMP_MI(t, ip, >);
7294 }
7295
7296 static inline void
7297 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
7298 {
7299         struct thread *t = &p->threads[p->thread_id];
7300         struct instruction *ip = t->ip;
7301
7302         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
7303
7304         JMP_CMP_HI(t, ip, >);
7305 }
7306
7307 /*
7308  * return.
7309  */
7310 static int
7311 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
7312                        struct action *action,
7313                        char **tokens __rte_unused,
7314                        int n_tokens,
7315                        struct instruction *instr,
7316                        struct instruction_data *data __rte_unused)
7317 {
7318         CHECK(action, EINVAL);
7319         CHECK(n_tokens == 1, EINVAL);
7320
7321         instr->type = INSTR_RETURN;
7322         return 0;
7323 }
7324
7325 static inline void
7326 instr_return_exec(struct rte_swx_pipeline *p)
7327 {
7328         struct thread *t = &p->threads[p->thread_id];
7329
7330         TRACE("[Thread %2u] return\n", p->thread_id);
7331
7332         t->ip = t->ret;
7333 }
7334
7335 static int
7336 instr_translate(struct rte_swx_pipeline *p,
7337                 struct action *action,
7338                 char *string,
7339                 struct instruction *instr,
7340                 struct instruction_data *data)
7341 {
7342         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
7343         int n_tokens = 0, tpos = 0;
7344
7345         /* Parse the instruction string into tokens. */
7346         for ( ; ; ) {
7347                 char *token;
7348
7349                 token = strtok_r(string, " \t\v", &string);
7350                 if (!token)
7351                         break;
7352
7353                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
7354                 CHECK_NAME(token, EINVAL);
7355
7356                 tokens[n_tokens] = token;
7357                 n_tokens++;
7358         }
7359
7360         CHECK(n_tokens, EINVAL);
7361
7362         /* Handle the optional instruction label. */
7363         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
7364                 strcpy(data->label, tokens[0]);
7365
7366                 tpos += 2;
7367                 CHECK(n_tokens - tpos, EINVAL);
7368         }
7369
7370         /* Identify the instruction type. */
7371         if (!strcmp(tokens[tpos], "rx"))
7372                 return instr_rx_translate(p,
7373                                           action,
7374                                           &tokens[tpos],
7375                                           n_tokens - tpos,
7376                                           instr,
7377                                           data);
7378
7379         if (!strcmp(tokens[tpos], "tx"))
7380                 return instr_tx_translate(p,
7381                                           action,
7382                                           &tokens[tpos],
7383                                           n_tokens - tpos,
7384                                           instr,
7385                                           data);
7386
7387         if (!strcmp(tokens[tpos], "drop"))
7388                 return instr_drop_translate(p,
7389                                             action,
7390                                             &tokens[tpos],
7391                                             n_tokens - tpos,
7392                                             instr,
7393                                             data);
7394
7395         if (!strcmp(tokens[tpos], "extract"))
7396                 return instr_hdr_extract_translate(p,
7397                                                    action,
7398                                                    &tokens[tpos],
7399                                                    n_tokens - tpos,
7400                                                    instr,
7401                                                    data);
7402
7403         if (!strcmp(tokens[tpos], "emit"))
7404                 return instr_hdr_emit_translate(p,
7405                                                 action,
7406                                                 &tokens[tpos],
7407                                                 n_tokens - tpos,
7408                                                 instr,
7409                                                 data);
7410
7411         if (!strcmp(tokens[tpos], "validate"))
7412                 return instr_hdr_validate_translate(p,
7413                                                     action,
7414                                                     &tokens[tpos],
7415                                                     n_tokens - tpos,
7416                                                     instr,
7417                                                     data);
7418
7419         if (!strcmp(tokens[tpos], "invalidate"))
7420                 return instr_hdr_invalidate_translate(p,
7421                                                       action,
7422                                                       &tokens[tpos],
7423                                                       n_tokens - tpos,
7424                                                       instr,
7425                                                       data);
7426
7427         if (!strcmp(tokens[tpos], "mov"))
7428                 return instr_mov_translate(p,
7429                                            action,
7430                                            &tokens[tpos],
7431                                            n_tokens - tpos,
7432                                            instr,
7433                                            data);
7434
7435         if (!strcmp(tokens[tpos], "dma"))
7436                 return instr_dma_translate(p,
7437                                            action,
7438                                            &tokens[tpos],
7439                                            n_tokens - tpos,
7440                                            instr,
7441                                            data);
7442
7443         if (!strcmp(tokens[tpos], "add"))
7444                 return instr_alu_add_translate(p,
7445                                                action,
7446                                                &tokens[tpos],
7447                                                n_tokens - tpos,
7448                                                instr,
7449                                                data);
7450
7451         if (!strcmp(tokens[tpos], "sub"))
7452                 return instr_alu_sub_translate(p,
7453                                                action,
7454                                                &tokens[tpos],
7455                                                n_tokens - tpos,
7456                                                instr,
7457                                                data);
7458
7459         if (!strcmp(tokens[tpos], "ckadd"))
7460                 return instr_alu_ckadd_translate(p,
7461                                                  action,
7462                                                  &tokens[tpos],
7463                                                  n_tokens - tpos,
7464                                                  instr,
7465                                                  data);
7466
7467         if (!strcmp(tokens[tpos], "cksub"))
7468                 return instr_alu_cksub_translate(p,
7469                                                  action,
7470                                                  &tokens[tpos],
7471                                                  n_tokens - tpos,
7472                                                  instr,
7473                                                  data);
7474
7475         if (!strcmp(tokens[tpos], "and"))
7476                 return instr_alu_and_translate(p,
7477                                                action,
7478                                                &tokens[tpos],
7479                                                n_tokens - tpos,
7480                                                instr,
7481                                                data);
7482
7483         if (!strcmp(tokens[tpos], "or"))
7484                 return instr_alu_or_translate(p,
7485                                               action,
7486                                               &tokens[tpos],
7487                                               n_tokens - tpos,
7488                                               instr,
7489                                               data);
7490
7491         if (!strcmp(tokens[tpos], "xor"))
7492                 return instr_alu_xor_translate(p,
7493                                                action,
7494                                                &tokens[tpos],
7495                                                n_tokens - tpos,
7496                                                instr,
7497                                                data);
7498
7499         if (!strcmp(tokens[tpos], "shl"))
7500                 return instr_alu_shl_translate(p,
7501                                                action,
7502                                                &tokens[tpos],
7503                                                n_tokens - tpos,
7504                                                instr,
7505                                                data);
7506
7507         if (!strcmp(tokens[tpos], "shr"))
7508                 return instr_alu_shr_translate(p,
7509                                                action,
7510                                                &tokens[tpos],
7511                                                n_tokens - tpos,
7512                                                instr,
7513                                                data);
7514
7515         if (!strcmp(tokens[tpos], "regprefetch"))
7516                 return instr_regprefetch_translate(p,
7517                                                    action,
7518                                                    &tokens[tpos],
7519                                                    n_tokens - tpos,
7520                                                    instr,
7521                                                    data);
7522
7523         if (!strcmp(tokens[tpos], "regrd"))
7524                 return instr_regrd_translate(p,
7525                                              action,
7526                                              &tokens[tpos],
7527                                              n_tokens - tpos,
7528                                              instr,
7529                                              data);
7530
7531         if (!strcmp(tokens[tpos], "regwr"))
7532                 return instr_regwr_translate(p,
7533                                              action,
7534                                              &tokens[tpos],
7535                                              n_tokens - tpos,
7536                                              instr,
7537                                              data);
7538
7539         if (!strcmp(tokens[tpos], "regadd"))
7540                 return instr_regadd_translate(p,
7541                                               action,
7542                                               &tokens[tpos],
7543                                               n_tokens - tpos,
7544                                               instr,
7545                                               data);
7546
7547         if (!strcmp(tokens[tpos], "metprefetch"))
7548                 return instr_metprefetch_translate(p,
7549                                                    action,
7550                                                    &tokens[tpos],
7551                                                    n_tokens - tpos,
7552                                                    instr,
7553                                                    data);
7554
7555         if (!strcmp(tokens[tpos], "meter"))
7556                 return instr_meter_translate(p,
7557                                              action,
7558                                              &tokens[tpos],
7559                                              n_tokens - tpos,
7560                                              instr,
7561                                              data);
7562
7563         if (!strcmp(tokens[tpos], "table"))
7564                 return instr_table_translate(p,
7565                                              action,
7566                                              &tokens[tpos],
7567                                              n_tokens - tpos,
7568                                              instr,
7569                                              data);
7570
7571         if (!strcmp(tokens[tpos], "extern"))
7572                 return instr_extern_translate(p,
7573                                               action,
7574                                               &tokens[tpos],
7575                                               n_tokens - tpos,
7576                                               instr,
7577                                               data);
7578
7579         if (!strcmp(tokens[tpos], "jmp"))
7580                 return instr_jmp_translate(p,
7581                                            action,
7582                                            &tokens[tpos],
7583                                            n_tokens - tpos,
7584                                            instr,
7585                                            data);
7586
7587         if (!strcmp(tokens[tpos], "jmpv"))
7588                 return instr_jmp_valid_translate(p,
7589                                                  action,
7590                                                  &tokens[tpos],
7591                                                  n_tokens - tpos,
7592                                                  instr,
7593                                                  data);
7594
7595         if (!strcmp(tokens[tpos], "jmpnv"))
7596                 return instr_jmp_invalid_translate(p,
7597                                                    action,
7598                                                    &tokens[tpos],
7599                                                    n_tokens - tpos,
7600                                                    instr,
7601                                                    data);
7602
7603         if (!strcmp(tokens[tpos], "jmph"))
7604                 return instr_jmp_hit_translate(p,
7605                                                action,
7606                                                &tokens[tpos],
7607                                                n_tokens - tpos,
7608                                                instr,
7609                                                data);
7610
7611         if (!strcmp(tokens[tpos], "jmpnh"))
7612                 return instr_jmp_miss_translate(p,
7613                                                 action,
7614                                                 &tokens[tpos],
7615                                                 n_tokens - tpos,
7616                                                 instr,
7617                                                 data);
7618
7619         if (!strcmp(tokens[tpos], "jmpa"))
7620                 return instr_jmp_action_hit_translate(p,
7621                                                       action,
7622                                                       &tokens[tpos],
7623                                                       n_tokens - tpos,
7624                                                       instr,
7625                                                       data);
7626
7627         if (!strcmp(tokens[tpos], "jmpna"))
7628                 return instr_jmp_action_miss_translate(p,
7629                                                        action,
7630                                                        &tokens[tpos],
7631                                                        n_tokens - tpos,
7632                                                        instr,
7633                                                        data);
7634
7635         if (!strcmp(tokens[tpos], "jmpeq"))
7636                 return instr_jmp_eq_translate(p,
7637                                               action,
7638                                               &tokens[tpos],
7639                                               n_tokens - tpos,
7640                                               instr,
7641                                               data);
7642
7643         if (!strcmp(tokens[tpos], "jmpneq"))
7644                 return instr_jmp_neq_translate(p,
7645                                                action,
7646                                                &tokens[tpos],
7647                                                n_tokens - tpos,
7648                                                instr,
7649                                                data);
7650
7651         if (!strcmp(tokens[tpos], "jmplt"))
7652                 return instr_jmp_lt_translate(p,
7653                                               action,
7654                                               &tokens[tpos],
7655                                               n_tokens - tpos,
7656                                               instr,
7657                                               data);
7658
7659         if (!strcmp(tokens[tpos], "jmpgt"))
7660                 return instr_jmp_gt_translate(p,
7661                                               action,
7662                                               &tokens[tpos],
7663                                               n_tokens - tpos,
7664                                               instr,
7665                                               data);
7666
7667         if (!strcmp(tokens[tpos], "return"))
7668                 return instr_return_translate(p,
7669                                               action,
7670                                               &tokens[tpos],
7671                                               n_tokens - tpos,
7672                                               instr,
7673                                               data);
7674
7675         CHECK(0, EINVAL);
7676 }
7677
7678 static struct instruction_data *
7679 label_find(struct instruction_data *data, uint32_t n, const char *label)
7680 {
7681         uint32_t i;
7682
7683         for (i = 0; i < n; i++)
7684                 if (!strcmp(label, data[i].label))
7685                         return &data[i];
7686
7687         return NULL;
7688 }
7689
7690 static uint32_t
7691 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
7692 {
7693         uint32_t count = 0, i;
7694
7695         if (!label[0])
7696                 return 0;
7697
7698         for (i = 0; i < n; i++)
7699                 if (!strcmp(label, data[i].jmp_label))
7700                         count++;
7701
7702         return count;
7703 }
7704
7705 static int
7706 instr_label_check(struct instruction_data *instruction_data,
7707                   uint32_t n_instructions)
7708 {
7709         uint32_t i;
7710
7711         /* Check that all instruction labels are unique. */
7712         for (i = 0; i < n_instructions; i++) {
7713                 struct instruction_data *data = &instruction_data[i];
7714                 char *label = data->label;
7715                 uint32_t j;
7716
7717                 if (!label[0])
7718                         continue;
7719
7720                 for (j = i + 1; j < n_instructions; j++)
7721                         CHECK(strcmp(label, data[j].label), EINVAL);
7722         }
7723
7724         /* Get users for each instruction label. */
7725         for (i = 0; i < n_instructions; i++) {
7726                 struct instruction_data *data = &instruction_data[i];
7727                 char *label = data->label;
7728
7729                 data->n_users = label_is_used(instruction_data,
7730                                               n_instructions,
7731                                               label);
7732         }
7733
7734         return 0;
7735 }
7736
7737 static int
7738 instr_jmp_resolve(struct instruction *instructions,
7739                   struct instruction_data *instruction_data,
7740                   uint32_t n_instructions)
7741 {
7742         uint32_t i;
7743
7744         for (i = 0; i < n_instructions; i++) {
7745                 struct instruction *instr = &instructions[i];
7746                 struct instruction_data *data = &instruction_data[i];
7747                 struct instruction_data *found;
7748
7749                 if (!instruction_is_jmp(instr))
7750                         continue;
7751
7752                 found = label_find(instruction_data,
7753                                    n_instructions,
7754                                    data->jmp_label);
7755                 CHECK(found, EINVAL);
7756
7757                 instr->jmp.ip = &instructions[found - instruction_data];
7758         }
7759
7760         return 0;
7761 }
7762
7763 static int
7764 instr_verify(struct rte_swx_pipeline *p __rte_unused,
7765              struct action *a,
7766              struct instruction *instr,
7767              struct instruction_data *data __rte_unused,
7768              uint32_t n_instructions)
7769 {
7770         if (!a) {
7771                 enum instruction_type type;
7772                 uint32_t i;
7773
7774                 /* Check that the first instruction is rx. */
7775                 CHECK(instr[0].type == INSTR_RX, EINVAL);
7776
7777                 /* Check that there is at least one tx instruction. */
7778                 for (i = 0; i < n_instructions; i++) {
7779                         type = instr[i].type;
7780
7781                         if (instruction_is_tx(type))
7782                                 break;
7783                 }
7784                 CHECK(i < n_instructions, EINVAL);
7785
7786                 /* Check that the last instruction is either tx or unconditional
7787                  * jump.
7788                  */
7789                 type = instr[n_instructions - 1].type;
7790                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
7791         }
7792
7793         if (a) {
7794                 enum instruction_type type;
7795                 uint32_t i;
7796
7797                 /* Check that there is at least one return or tx instruction. */
7798                 for (i = 0; i < n_instructions; i++) {
7799                         type = instr[i].type;
7800
7801                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
7802                                 break;
7803                 }
7804                 CHECK(i < n_instructions, EINVAL);
7805         }
7806
7807         return 0;
7808 }
7809
7810 static int
7811 instr_pattern_extract_many_detect(struct instruction *instr,
7812                                   struct instruction_data *data,
7813                                   uint32_t n_instr,
7814                                   uint32_t *n_pattern_instr)
7815 {
7816         uint32_t i;
7817
7818         for (i = 0; i < n_instr; i++) {
7819                 if (data[i].invalid)
7820                         break;
7821
7822                 if (instr[i].type != INSTR_HDR_EXTRACT)
7823                         break;
7824
7825                 if (i == RTE_DIM(instr->io.hdr.header_id))
7826                         break;
7827
7828                 if (i && data[i].n_users)
7829                         break;
7830         }
7831
7832         if (i < 2)
7833                 return 0;
7834
7835         *n_pattern_instr = i;
7836         return 1;
7837 }
7838
7839 static void
7840 instr_pattern_extract_many_optimize(struct instruction *instr,
7841                                     struct instruction_data *data,
7842                                     uint32_t n_instr)
7843 {
7844         uint32_t i;
7845
7846         for (i = 1; i < n_instr; i++) {
7847                 instr[0].type++;
7848                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7849                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7850                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7851
7852                 data[i].invalid = 1;
7853         }
7854 }
7855
7856 static int
7857 instr_pattern_emit_many_tx_detect(struct instruction *instr,
7858                                   struct instruction_data *data,
7859                                   uint32_t n_instr,
7860                                   uint32_t *n_pattern_instr)
7861 {
7862         uint32_t i;
7863
7864         for (i = 0; i < n_instr; i++) {
7865                 if (data[i].invalid)
7866                         break;
7867
7868                 if (instr[i].type != INSTR_HDR_EMIT)
7869                         break;
7870
7871                 if (i == RTE_DIM(instr->io.hdr.header_id))
7872                         break;
7873
7874                 if (i && data[i].n_users)
7875                         break;
7876         }
7877
7878         if (!i)
7879                 return 0;
7880
7881         if (!instruction_is_tx(instr[i].type))
7882                 return 0;
7883
7884         if (data[i].n_users)
7885                 return 0;
7886
7887         i++;
7888
7889         *n_pattern_instr = i;
7890         return 1;
7891 }
7892
7893 static void
7894 instr_pattern_emit_many_tx_optimize(struct instruction *instr,
7895                                     struct instruction_data *data,
7896                                     uint32_t n_instr)
7897 {
7898         uint32_t i;
7899
7900         /* Any emit instruction in addition to the first one. */
7901         for (i = 1; i < n_instr - 1; i++) {
7902                 instr[0].type++;
7903                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7904                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7905                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7906
7907                 data[i].invalid = 1;
7908         }
7909
7910         /* The TX instruction is the last one in the pattern. */
7911         instr[0].type++;
7912         instr[0].io.io.offset = instr[i].io.io.offset;
7913         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7914         data[i].invalid = 1;
7915 }
7916
7917 static int
7918 instr_pattern_dma_many_detect(struct instruction *instr,
7919                               struct instruction_data *data,
7920                               uint32_t n_instr,
7921                               uint32_t *n_pattern_instr)
7922 {
7923         uint32_t i;
7924
7925         for (i = 0; i < n_instr; i++) {
7926                 if (data[i].invalid)
7927                         break;
7928
7929                 if (instr[i].type != INSTR_DMA_HT)
7930                         break;
7931
7932                 if (i == RTE_DIM(instr->dma.dst.header_id))
7933                         break;
7934
7935                 if (i && data[i].n_users)
7936                         break;
7937         }
7938
7939         if (i < 2)
7940                 return 0;
7941
7942         *n_pattern_instr = i;
7943         return 1;
7944 }
7945
7946 static void
7947 instr_pattern_dma_many_optimize(struct instruction *instr,
7948                                 struct instruction_data *data,
7949                                 uint32_t n_instr)
7950 {
7951         uint32_t i;
7952
7953         for (i = 1; i < n_instr; i++) {
7954                 instr[0].type++;
7955                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7956                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7957                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7958                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7959
7960                 data[i].invalid = 1;
7961         }
7962 }
7963
7964 static uint32_t
7965 instr_optimize(struct instruction *instructions,
7966                struct instruction_data *instruction_data,
7967                uint32_t n_instructions)
7968 {
7969         uint32_t i, pos = 0;
7970
7971         for (i = 0; i < n_instructions; ) {
7972                 struct instruction *instr = &instructions[i];
7973                 struct instruction_data *data = &instruction_data[i];
7974                 uint32_t n_instr = 0;
7975                 int detected;
7976
7977                 /* Extract many. */
7978                 detected = instr_pattern_extract_many_detect(instr,
7979                                                              data,
7980                                                              n_instructions - i,
7981                                                              &n_instr);
7982                 if (detected) {
7983                         instr_pattern_extract_many_optimize(instr,
7984                                                             data,
7985                                                             n_instr);
7986                         i += n_instr;
7987                         continue;
7988                 }
7989
7990                 /* Emit many + TX. */
7991                 detected = instr_pattern_emit_many_tx_detect(instr,
7992                                                              data,
7993                                                              n_instructions - i,
7994                                                              &n_instr);
7995                 if (detected) {
7996                         instr_pattern_emit_many_tx_optimize(instr,
7997                                                             data,
7998                                                             n_instr);
7999                         i += n_instr;
8000                         continue;
8001                 }
8002
8003                 /* DMA many. */
8004                 detected = instr_pattern_dma_many_detect(instr,
8005                                                          data,
8006                                                          n_instructions - i,
8007                                                          &n_instr);
8008                 if (detected) {
8009                         instr_pattern_dma_many_optimize(instr, data, n_instr);
8010                         i += n_instr;
8011                         continue;
8012                 }
8013
8014                 /* No pattern starting at the current instruction. */
8015                 i++;
8016         }
8017
8018         /* Eliminate the invalid instructions that have been optimized out. */
8019         for (i = 0; i < n_instructions; i++) {
8020                 struct instruction *instr = &instructions[i];
8021                 struct instruction_data *data = &instruction_data[i];
8022
8023                 if (data->invalid)
8024                         continue;
8025
8026                 if (i != pos) {
8027                         memcpy(&instructions[pos], instr, sizeof(*instr));
8028                         memcpy(&instruction_data[pos], data, sizeof(*data));
8029                 }
8030
8031                 pos++;
8032         }
8033
8034         return pos;
8035 }
8036
8037 static int
8038 instruction_config(struct rte_swx_pipeline *p,
8039                    struct action *a,
8040                    const char **instructions,
8041                    uint32_t n_instructions)
8042 {
8043         struct instruction *instr = NULL;
8044         struct instruction_data *data = NULL;
8045         int err = 0;
8046         uint32_t i;
8047
8048         CHECK(n_instructions, EINVAL);
8049         CHECK(instructions, EINVAL);
8050         for (i = 0; i < n_instructions; i++)
8051                 CHECK_INSTRUCTION(instructions[i], EINVAL);
8052
8053         /* Memory allocation. */
8054         instr = calloc(n_instructions, sizeof(struct instruction));
8055         if (!instr) {
8056                 err = ENOMEM;
8057                 goto error;
8058         }
8059
8060         data = calloc(n_instructions, sizeof(struct instruction_data));
8061         if (!data) {
8062                 err = ENOMEM;
8063                 goto error;
8064         }
8065
8066         for (i = 0; i < n_instructions; i++) {
8067                 char *string = strdup(instructions[i]);
8068                 if (!string) {
8069                         err = ENOMEM;
8070                         goto error;
8071                 }
8072
8073                 err = instr_translate(p, a, string, &instr[i], &data[i]);
8074                 if (err) {
8075                         free(string);
8076                         goto error;
8077                 }
8078
8079                 free(string);
8080         }
8081
8082         err = instr_label_check(data, n_instructions);
8083         if (err)
8084                 goto error;
8085
8086         err = instr_verify(p, a, instr, data, n_instructions);
8087         if (err)
8088                 goto error;
8089
8090         n_instructions = instr_optimize(instr, data, n_instructions);
8091
8092         err = instr_jmp_resolve(instr, data, n_instructions);
8093         if (err)
8094                 goto error;
8095
8096         if (a) {
8097                 a->instructions = instr;
8098                 a->n_instructions = n_instructions;
8099         } else {
8100                 p->instructions = instr;
8101                 p->n_instructions = n_instructions;
8102         }
8103
8104         free(data);
8105         return 0;
8106
8107 error:
8108         free(data);
8109         free(instr);
8110         return err;
8111 }
8112
8113 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
8114
8115 static instr_exec_t instruction_table[] = {
8116         [INSTR_RX] = instr_rx_exec,
8117         [INSTR_TX] = instr_tx_exec,
8118         [INSTR_TX_I] = instr_tx_i_exec,
8119
8120         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
8121         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
8122         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
8123         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
8124         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
8125         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
8126         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
8127         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
8128
8129         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
8130         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
8131         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
8132         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
8133         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
8134         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
8135         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
8136         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
8137         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
8138
8139         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
8140         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
8141
8142         [INSTR_MOV] = instr_mov_exec,
8143         [INSTR_MOV_S] = instr_mov_s_exec,
8144         [INSTR_MOV_I] = instr_mov_i_exec,
8145
8146         [INSTR_DMA_HT] = instr_dma_ht_exec,
8147         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
8148         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
8149         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
8150         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
8151         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
8152         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
8153         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
8154
8155         [INSTR_ALU_ADD] = instr_alu_add_exec,
8156         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
8157         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
8158         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
8159         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
8160         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
8161
8162         [INSTR_ALU_SUB] = instr_alu_sub_exec,
8163         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
8164         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
8165         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
8166         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
8167         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
8168
8169         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
8170         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
8171         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
8172         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
8173
8174         [INSTR_ALU_AND] = instr_alu_and_exec,
8175         [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
8176         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
8177
8178         [INSTR_ALU_OR] = instr_alu_or_exec,
8179         [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
8180         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
8181
8182         [INSTR_ALU_XOR] = instr_alu_xor_exec,
8183         [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec,
8184         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
8185
8186         [INSTR_ALU_SHL] = instr_alu_shl_exec,
8187         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
8188         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
8189         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
8190         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
8191         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
8192
8193         [INSTR_ALU_SHR] = instr_alu_shr_exec,
8194         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
8195         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
8196         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
8197         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
8198         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
8199
8200         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
8201         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
8202         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
8203
8204         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
8205         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
8206         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
8207         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
8208         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
8209         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
8210
8211         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
8212         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
8213         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
8214         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
8215         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
8216         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
8217         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
8218         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
8219         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
8220
8221         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
8222         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
8223         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
8224         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
8225         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
8226         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
8227         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
8228         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
8229         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
8230
8231         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
8232         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
8233         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
8234
8235         [INSTR_METER_HHM] = instr_meter_hhm_exec,
8236         [INSTR_METER_HHI] = instr_meter_hhi_exec,
8237         [INSTR_METER_HMM] = instr_meter_hmm_exec,
8238         [INSTR_METER_HMI] = instr_meter_hmi_exec,
8239         [INSTR_METER_MHM] = instr_meter_mhm_exec,
8240         [INSTR_METER_MHI] = instr_meter_mhi_exec,
8241         [INSTR_METER_MMM] = instr_meter_mmm_exec,
8242         [INSTR_METER_MMI] = instr_meter_mmi_exec,
8243         [INSTR_METER_IHM] = instr_meter_ihm_exec,
8244         [INSTR_METER_IHI] = instr_meter_ihi_exec,
8245         [INSTR_METER_IMM] = instr_meter_imm_exec,
8246         [INSTR_METER_IMI] = instr_meter_imi_exec,
8247
8248         [INSTR_TABLE] = instr_table_exec,
8249         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
8250         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
8251
8252         [INSTR_JMP] = instr_jmp_exec,
8253         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
8254         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
8255         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
8256         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
8257         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
8258         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
8259
8260         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
8261         [INSTR_JMP_EQ_S] = instr_jmp_eq_s_exec,
8262         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
8263
8264         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
8265         [INSTR_JMP_NEQ_S] = instr_jmp_neq_s_exec,
8266         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
8267
8268         [INSTR_JMP_LT] = instr_jmp_lt_exec,
8269         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
8270         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
8271         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
8272         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
8273         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
8274
8275         [INSTR_JMP_GT] = instr_jmp_gt_exec,
8276         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
8277         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
8278         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
8279         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
8280         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
8281
8282         [INSTR_RETURN] = instr_return_exec,
8283 };
8284
8285 static inline void
8286 instr_exec(struct rte_swx_pipeline *p)
8287 {
8288         struct thread *t = &p->threads[p->thread_id];
8289         struct instruction *ip = t->ip;
8290         instr_exec_t instr = instruction_table[ip->type];
8291
8292         instr(p);
8293 }
8294
8295 /*
8296  * Action.
8297  */
8298 static struct action *
8299 action_find(struct rte_swx_pipeline *p, const char *name)
8300 {
8301         struct action *elem;
8302
8303         if (!name)
8304                 return NULL;
8305
8306         TAILQ_FOREACH(elem, &p->actions, node)
8307                 if (strcmp(elem->name, name) == 0)
8308                         return elem;
8309
8310         return NULL;
8311 }
8312
8313 static struct action *
8314 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8315 {
8316         struct action *action = NULL;
8317
8318         TAILQ_FOREACH(action, &p->actions, node)
8319                 if (action->id == id)
8320                         return action;
8321
8322         return NULL;
8323 }
8324
8325 static struct field *
8326 action_field_find(struct action *a, const char *name)
8327 {
8328         return a->st ? struct_type_field_find(a->st, name) : NULL;
8329 }
8330
8331 static struct field *
8332 action_field_parse(struct action *action, const char *name)
8333 {
8334         if (name[0] != 't' || name[1] != '.')
8335                 return NULL;
8336
8337         return action_field_find(action, &name[2]);
8338 }
8339
8340 int
8341 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
8342                                const char *name,
8343                                const char *args_struct_type_name,
8344                                const char **instructions,
8345                                uint32_t n_instructions)
8346 {
8347         struct struct_type *args_struct_type;
8348         struct action *a;
8349         int err;
8350
8351         CHECK(p, EINVAL);
8352
8353         CHECK_NAME(name, EINVAL);
8354         CHECK(!action_find(p, name), EEXIST);
8355
8356         if (args_struct_type_name) {
8357                 CHECK_NAME(args_struct_type_name, EINVAL);
8358                 args_struct_type = struct_type_find(p, args_struct_type_name);
8359                 CHECK(args_struct_type, EINVAL);
8360         } else {
8361                 args_struct_type = NULL;
8362         }
8363
8364         /* Node allocation. */
8365         a = calloc(1, sizeof(struct action));
8366         CHECK(a, ENOMEM);
8367
8368         /* Node initialization. */
8369         strcpy(a->name, name);
8370         a->st = args_struct_type;
8371         a->id = p->n_actions;
8372
8373         /* Instruction translation. */
8374         err = instruction_config(p, a, instructions, n_instructions);
8375         if (err) {
8376                 free(a);
8377                 return err;
8378         }
8379
8380         /* Node add to tailq. */
8381         TAILQ_INSERT_TAIL(&p->actions, a, node);
8382         p->n_actions++;
8383
8384         return 0;
8385 }
8386
8387 static int
8388 action_build(struct rte_swx_pipeline *p)
8389 {
8390         struct action *action;
8391
8392         p->action_instructions = calloc(p->n_actions,
8393                                         sizeof(struct instruction *));
8394         CHECK(p->action_instructions, ENOMEM);
8395
8396         TAILQ_FOREACH(action, &p->actions, node)
8397                 p->action_instructions[action->id] = action->instructions;
8398
8399         return 0;
8400 }
8401
8402 static void
8403 action_build_free(struct rte_swx_pipeline *p)
8404 {
8405         free(p->action_instructions);
8406         p->action_instructions = NULL;
8407 }
8408
8409 static void
8410 action_free(struct rte_swx_pipeline *p)
8411 {
8412         action_build_free(p);
8413
8414         for ( ; ; ) {
8415                 struct action *action;
8416
8417                 action = TAILQ_FIRST(&p->actions);
8418                 if (!action)
8419                         break;
8420
8421                 TAILQ_REMOVE(&p->actions, action, node);
8422                 free(action->instructions);
8423                 free(action);
8424         }
8425 }
8426
8427 /*
8428  * Table.
8429  */
8430 static struct table_type *
8431 table_type_find(struct rte_swx_pipeline *p, const char *name)
8432 {
8433         struct table_type *elem;
8434
8435         TAILQ_FOREACH(elem, &p->table_types, node)
8436                 if (strcmp(elem->name, name) == 0)
8437                         return elem;
8438
8439         return NULL;
8440 }
8441
8442 static struct table_type *
8443 table_type_resolve(struct rte_swx_pipeline *p,
8444                    const char *recommended_type_name,
8445                    enum rte_swx_table_match_type match_type)
8446 {
8447         struct table_type *elem;
8448
8449         /* Only consider the recommended type if the match type is correct. */
8450         if (recommended_type_name)
8451                 TAILQ_FOREACH(elem, &p->table_types, node)
8452                         if (!strcmp(elem->name, recommended_type_name) &&
8453                             (elem->match_type == match_type))
8454                                 return elem;
8455
8456         /* Ignore the recommended type and get the first element with this match
8457          * type.
8458          */
8459         TAILQ_FOREACH(elem, &p->table_types, node)
8460                 if (elem->match_type == match_type)
8461                         return elem;
8462
8463         return NULL;
8464 }
8465
8466 static struct table *
8467 table_find(struct rte_swx_pipeline *p, const char *name)
8468 {
8469         struct table *elem;
8470
8471         TAILQ_FOREACH(elem, &p->tables, node)
8472                 if (strcmp(elem->name, name) == 0)
8473                         return elem;
8474
8475         return NULL;
8476 }
8477
8478 static struct table *
8479 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8480 {
8481         struct table *table = NULL;
8482
8483         TAILQ_FOREACH(table, &p->tables, node)
8484                 if (table->id == id)
8485                         return table;
8486
8487         return NULL;
8488 }
8489
8490 int
8491 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
8492                                      const char *name,
8493                                      enum rte_swx_table_match_type match_type,
8494                                      struct rte_swx_table_ops *ops)
8495 {
8496         struct table_type *elem;
8497
8498         CHECK(p, EINVAL);
8499
8500         CHECK_NAME(name, EINVAL);
8501         CHECK(!table_type_find(p, name), EEXIST);
8502
8503         CHECK(ops, EINVAL);
8504         CHECK(ops->create, EINVAL);
8505         CHECK(ops->lkp, EINVAL);
8506         CHECK(ops->free, EINVAL);
8507
8508         /* Node allocation. */
8509         elem = calloc(1, sizeof(struct table_type));
8510         CHECK(elem, ENOMEM);
8511
8512         /* Node initialization. */
8513         strcpy(elem->name, name);
8514         elem->match_type = match_type;
8515         memcpy(&elem->ops, ops, sizeof(*ops));
8516
8517         /* Node add to tailq. */
8518         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
8519
8520         return 0;
8521 }
8522
8523 static enum rte_swx_table_match_type
8524 table_match_type_resolve(struct rte_swx_match_field_params *fields,
8525                          uint32_t n_fields,
8526                          uint32_t max_offset_field_id)
8527 {
8528         uint32_t n_fields_em = 0, i;
8529
8530         for (i = 0; i < n_fields; i++)
8531                 if (fields[i].match_type == RTE_SWX_TABLE_MATCH_EXACT)
8532                         n_fields_em++;
8533
8534         if (n_fields_em == n_fields)
8535                 return RTE_SWX_TABLE_MATCH_EXACT;
8536
8537         if ((n_fields_em == n_fields - 1) &&
8538             (fields[max_offset_field_id].match_type == RTE_SWX_TABLE_MATCH_LPM))
8539                 return RTE_SWX_TABLE_MATCH_LPM;
8540
8541         return RTE_SWX_TABLE_MATCH_WILDCARD;
8542 }
8543
8544 static int
8545 table_match_fields_check(struct rte_swx_pipeline *p,
8546                          struct rte_swx_pipeline_table_params *params,
8547                          struct header **header,
8548                          uint32_t *min_offset_field_id,
8549                          uint32_t *max_offset_field_id)
8550 {
8551         struct header *h0 = NULL;
8552         struct field *hf, *mf;
8553         uint32_t *offset = NULL, min_offset, max_offset, min_offset_pos, max_offset_pos, i;
8554         int status = 0;
8555
8556         /* Return if no match fields. */
8557         if (!params->n_fields) {
8558                 if (params->fields) {
8559                         status = -EINVAL;
8560                         goto end;
8561                 }
8562
8563                 return 0;
8564         }
8565
8566         /* Memory allocation. */
8567         offset = calloc(params->n_fields, sizeof(uint32_t));
8568         if (!offset) {
8569                 status = -ENOMEM;
8570                 goto end;
8571         }
8572
8573         /* Check that all the match fields belong to either the same header or
8574          * to the meta-data.
8575          */
8576         hf = header_field_parse(p, params->fields[0].name, &h0);
8577         mf = metadata_field_parse(p, params->fields[0].name);
8578         if (!hf && !mf) {
8579                 status = -EINVAL;
8580                 goto end;
8581         }
8582
8583         offset[0] = h0 ? hf->offset : mf->offset;
8584
8585         for (i = 1; i < params->n_fields; i++)
8586                 if (h0) {
8587                         struct header *h;
8588
8589                         hf = header_field_parse(p, params->fields[i].name, &h);
8590                         if (!hf || (h->id != h0->id)) {
8591                                 status = -EINVAL;
8592                                 goto end;
8593                         }
8594
8595                         offset[i] = hf->offset;
8596                 } else {
8597                         mf = metadata_field_parse(p, params->fields[i].name);
8598                         if (!mf) {
8599                                 status = -EINVAL;
8600                                 goto end;
8601                         }
8602
8603                         offset[i] = mf->offset;
8604                 }
8605
8606         /* Check that there are no duplicated match fields. */
8607         for (i = 0; i < params->n_fields; i++) {
8608                 uint32_t j;
8609
8610                 for (j = 0; j < i; j++)
8611                         if (offset[j] == offset[i]) {
8612                                 status = -EINVAL;
8613                                 goto end;
8614                         }
8615         }
8616
8617         /* Find the min and max offset fields. */
8618         min_offset = offset[0];
8619         max_offset = offset[0];
8620         min_offset_pos = 0;
8621         max_offset_pos = 0;
8622
8623         for (i = 1; i < params->n_fields; i++) {
8624                 if (offset[i] < min_offset) {
8625                         min_offset = offset[i];
8626                         min_offset_pos = i;
8627                 }
8628
8629                 if (offset[i] > max_offset) {
8630                         max_offset = offset[i];
8631                         max_offset_pos = i;
8632                 }
8633         }
8634
8635         /* Return. */
8636         if (header)
8637                 *header = h0;
8638
8639         if (min_offset_field_id)
8640                 *min_offset_field_id = min_offset_pos;
8641
8642         if (max_offset_field_id)
8643                 *max_offset_field_id = max_offset_pos;
8644
8645 end:
8646         free(offset);
8647         return status;
8648 }
8649
8650 int
8651 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8652                               const char *name,
8653                               struct rte_swx_pipeline_table_params *params,
8654                               const char *recommended_table_type_name,
8655                               const char *args,
8656                               uint32_t size)
8657 {
8658         struct table_type *type;
8659         struct table *t;
8660         struct action *default_action;
8661         struct header *header = NULL;
8662         uint32_t action_data_size_max = 0, min_offset_field_id = 0, max_offset_field_id = 0, i;
8663         int status = 0;
8664
8665         CHECK(p, EINVAL);
8666
8667         CHECK_NAME(name, EINVAL);
8668         CHECK(!table_find(p, name), EEXIST);
8669
8670         CHECK(params, EINVAL);
8671
8672         /* Match checks. */
8673         status = table_match_fields_check(p,
8674                                           params,
8675                                           &header,
8676                                           &min_offset_field_id,
8677                                           &max_offset_field_id);
8678         if (status)
8679                 return status;
8680
8681         /* Action checks. */
8682         CHECK(params->n_actions, EINVAL);
8683         CHECK(params->action_names, EINVAL);
8684         for (i = 0; i < params->n_actions; i++) {
8685                 const char *action_name = params->action_names[i];
8686                 struct action *a;
8687                 uint32_t action_data_size;
8688
8689                 CHECK_NAME(action_name, EINVAL);
8690
8691                 a = action_find(p, action_name);
8692                 CHECK(a, EINVAL);
8693
8694                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8695                 if (action_data_size > action_data_size_max)
8696                         action_data_size_max = action_data_size;
8697         }
8698
8699         CHECK_NAME(params->default_action_name, EINVAL);
8700         for (i = 0; i < p->n_actions; i++)
8701                 if (!strcmp(params->action_names[i],
8702                             params->default_action_name))
8703                         break;
8704         CHECK(i < params->n_actions, EINVAL);
8705         default_action = action_find(p, params->default_action_name);
8706         CHECK((default_action->st && params->default_action_data) ||
8707               !params->default_action_data, EINVAL);
8708
8709         /* Table type checks. */
8710         if (recommended_table_type_name)
8711                 CHECK_NAME(recommended_table_type_name, EINVAL);
8712
8713         if (params->n_fields) {
8714                 enum rte_swx_table_match_type match_type;
8715
8716                 match_type = table_match_type_resolve(params->fields,
8717                                                       params->n_fields,
8718                                                       max_offset_field_id);
8719                 type = table_type_resolve(p,
8720                                           recommended_table_type_name,
8721                                           match_type);
8722                 CHECK(type, EINVAL);
8723         } else {
8724                 type = NULL;
8725         }
8726
8727         /* Memory allocation. */
8728         t = calloc(1, sizeof(struct table));
8729         CHECK(t, ENOMEM);
8730
8731         t->fields = calloc(params->n_fields, sizeof(struct match_field));
8732         if (!t->fields) {
8733                 free(t);
8734                 CHECK(0, ENOMEM);
8735         }
8736
8737         t->actions = calloc(params->n_actions, sizeof(struct action *));
8738         if (!t->actions) {
8739                 free(t->fields);
8740                 free(t);
8741                 CHECK(0, ENOMEM);
8742         }
8743
8744         if (action_data_size_max) {
8745                 t->default_action_data = calloc(1, action_data_size_max);
8746                 if (!t->default_action_data) {
8747                         free(t->actions);
8748                         free(t->fields);
8749                         free(t);
8750                         CHECK(0, ENOMEM);
8751                 }
8752         }
8753
8754         /* Node initialization. */
8755         strcpy(t->name, name);
8756         if (args && args[0])
8757                 strcpy(t->args, args);
8758         t->type = type;
8759
8760         for (i = 0; i < params->n_fields; i++) {
8761                 struct rte_swx_match_field_params *field = &params->fields[i];
8762                 struct match_field *f = &t->fields[i];
8763
8764                 f->match_type = field->match_type;
8765                 f->field = header ?
8766                         header_field_parse(p, field->name, NULL) :
8767                         metadata_field_parse(p, field->name);
8768         }
8769         t->n_fields = params->n_fields;
8770         t->header = header;
8771
8772         for (i = 0; i < params->n_actions; i++)
8773                 t->actions[i] = action_find(p, params->action_names[i]);
8774         t->default_action = default_action;
8775         if (default_action->st)
8776                 memcpy(t->default_action_data,
8777                        params->default_action_data,
8778                        default_action->st->n_bits / 8);
8779         t->n_actions = params->n_actions;
8780         t->default_action_is_const = params->default_action_is_const;
8781         t->action_data_size_max = action_data_size_max;
8782
8783         t->size = size;
8784         t->id = p->n_tables;
8785
8786         /* Node add to tailq. */
8787         TAILQ_INSERT_TAIL(&p->tables, t, node);
8788         p->n_tables++;
8789
8790         return 0;
8791 }
8792
8793 static struct rte_swx_table_params *
8794 table_params_get(struct table *table)
8795 {
8796         struct rte_swx_table_params *params;
8797         struct field *first, *last;
8798         uint8_t *key_mask;
8799         uint32_t key_size, key_offset, action_data_size, i;
8800
8801         /* Memory allocation. */
8802         params = calloc(1, sizeof(struct rte_swx_table_params));
8803         if (!params)
8804                 return NULL;
8805
8806         /* Find first (smallest offset) and last (biggest offset) match fields. */
8807         first = table->fields[0].field;
8808         last = table->fields[0].field;
8809
8810         for (i = 0; i < table->n_fields; i++) {
8811                 struct field *f = table->fields[i].field;
8812
8813                 if (f->offset < first->offset)
8814                         first = f;
8815
8816                 if (f->offset > last->offset)
8817                         last = f;
8818         }
8819
8820         /* Key offset and size. */
8821         key_offset = first->offset / 8;
8822         key_size = (last->offset + last->n_bits - first->offset) / 8;
8823
8824         /* Memory allocation. */
8825         key_mask = calloc(1, key_size);
8826         if (!key_mask) {
8827                 free(params);
8828                 return NULL;
8829         }
8830
8831         /* Key mask. */
8832         for (i = 0; i < table->n_fields; i++) {
8833                 struct field *f = table->fields[i].field;
8834                 uint32_t start = (f->offset - first->offset) / 8;
8835                 size_t size = f->n_bits / 8;
8836
8837                 memset(&key_mask[start], 0xFF, size);
8838         }
8839
8840         /* Action data size. */
8841         action_data_size = 0;
8842         for (i = 0; i < table->n_actions; i++) {
8843                 struct action *action = table->actions[i];
8844                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8845
8846                 if (ads > action_data_size)
8847                         action_data_size = ads;
8848         }
8849
8850         /* Fill in. */
8851         params->match_type = table->type->match_type;
8852         params->key_size = key_size;
8853         params->key_offset = key_offset;
8854         params->key_mask0 = key_mask;
8855         params->action_data_size = action_data_size;
8856         params->n_keys_max = table->size;
8857
8858         return params;
8859 }
8860
8861 static void
8862 table_params_free(struct rte_swx_table_params *params)
8863 {
8864         if (!params)
8865                 return;
8866
8867         free(params->key_mask0);
8868         free(params);
8869 }
8870
8871 static int
8872 table_state_build(struct rte_swx_pipeline *p)
8873 {
8874         struct table *table;
8875
8876         p->table_state = calloc(p->n_tables,
8877                                 sizeof(struct rte_swx_table_state));
8878         CHECK(p->table_state, ENOMEM);
8879
8880         TAILQ_FOREACH(table, &p->tables, node) {
8881                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8882
8883                 if (table->type) {
8884                         struct rte_swx_table_params *params;
8885
8886                         /* ts->obj. */
8887                         params = table_params_get(table);
8888                         CHECK(params, ENOMEM);
8889
8890                         ts->obj = table->type->ops.create(params,
8891                                 NULL,
8892                                 table->args,
8893                                 p->numa_node);
8894
8895                         table_params_free(params);
8896                         CHECK(ts->obj, ENODEV);
8897                 }
8898
8899                 /* ts->default_action_data. */
8900                 if (table->action_data_size_max) {
8901                         ts->default_action_data =
8902                                 malloc(table->action_data_size_max);
8903                         CHECK(ts->default_action_data, ENOMEM);
8904
8905                         memcpy(ts->default_action_data,
8906                                table->default_action_data,
8907                                table->action_data_size_max);
8908                 }
8909
8910                 /* ts->default_action_id. */
8911                 ts->default_action_id = table->default_action->id;
8912         }
8913
8914         return 0;
8915 }
8916
8917 static void
8918 table_state_build_free(struct rte_swx_pipeline *p)
8919 {
8920         uint32_t i;
8921
8922         if (!p->table_state)
8923                 return;
8924
8925         for (i = 0; i < p->n_tables; i++) {
8926                 struct rte_swx_table_state *ts = &p->table_state[i];
8927                 struct table *table = table_find_by_id(p, i);
8928
8929                 /* ts->obj. */
8930                 if (table->type && ts->obj)
8931                         table->type->ops.free(ts->obj);
8932
8933                 /* ts->default_action_data. */
8934                 free(ts->default_action_data);
8935         }
8936
8937         free(p->table_state);
8938         p->table_state = NULL;
8939 }
8940
8941 static void
8942 table_state_free(struct rte_swx_pipeline *p)
8943 {
8944         table_state_build_free(p);
8945 }
8946
8947 static int
8948 table_stub_lkp(void *table __rte_unused,
8949                void *mailbox __rte_unused,
8950                uint8_t **key __rte_unused,
8951                uint64_t *action_id __rte_unused,
8952                uint8_t **action_data __rte_unused,
8953                int *hit)
8954 {
8955         *hit = 0;
8956         return 1; /* DONE. */
8957 }
8958
8959 static int
8960 table_build(struct rte_swx_pipeline *p)
8961 {
8962         uint32_t i;
8963
8964         /* Per pipeline: table statistics. */
8965         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8966         CHECK(p->table_stats, ENOMEM);
8967
8968         for (i = 0; i < p->n_tables; i++) {
8969                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8970                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8971         }
8972
8973         /* Per thread: table runt-time. */
8974         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8975                 struct thread *t = &p->threads[i];
8976                 struct table *table;
8977
8978                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8979                 CHECK(t->tables, ENOMEM);
8980
8981                 TAILQ_FOREACH(table, &p->tables, node) {
8982                         struct table_runtime *r = &t->tables[table->id];
8983
8984                         if (table->type) {
8985                                 uint64_t size;
8986
8987                                 size = table->type->ops.mailbox_size_get();
8988
8989                                 /* r->func. */
8990                                 r->func = table->type->ops.lkp;
8991
8992                                 /* r->mailbox. */
8993                                 if (size) {
8994                                         r->mailbox = calloc(1, size);
8995                                         CHECK(r->mailbox, ENOMEM);
8996                                 }
8997
8998                                 /* r->key. */
8999                                 r->key = table->header ?
9000                                         &t->structs[table->header->struct_id] :
9001                                         &t->structs[p->metadata_struct_id];
9002                         } else {
9003                                 r->func = table_stub_lkp;
9004                         }
9005                 }
9006         }
9007
9008         return 0;
9009 }
9010
9011 static void
9012 table_build_free(struct rte_swx_pipeline *p)
9013 {
9014         uint32_t i;
9015
9016         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9017                 struct thread *t = &p->threads[i];
9018                 uint32_t j;
9019
9020                 if (!t->tables)
9021                         continue;
9022
9023                 for (j = 0; j < p->n_tables; j++) {
9024                         struct table_runtime *r = &t->tables[j];
9025
9026                         free(r->mailbox);
9027                 }
9028
9029                 free(t->tables);
9030                 t->tables = NULL;
9031         }
9032
9033         if (p->table_stats) {
9034                 for (i = 0; i < p->n_tables; i++)
9035                         free(p->table_stats[i].n_pkts_action);
9036
9037                 free(p->table_stats);
9038         }
9039 }
9040
9041 static void
9042 table_free(struct rte_swx_pipeline *p)
9043 {
9044         table_build_free(p);
9045
9046         /* Tables. */
9047         for ( ; ; ) {
9048                 struct table *elem;
9049
9050                 elem = TAILQ_FIRST(&p->tables);
9051                 if (!elem)
9052                         break;
9053
9054                 TAILQ_REMOVE(&p->tables, elem, node);
9055                 free(elem->fields);
9056                 free(elem->actions);
9057                 free(elem->default_action_data);
9058                 free(elem);
9059         }
9060
9061         /* Table types. */
9062         for ( ; ; ) {
9063                 struct table_type *elem;
9064
9065                 elem = TAILQ_FIRST(&p->table_types);
9066                 if (!elem)
9067                         break;
9068
9069                 TAILQ_REMOVE(&p->table_types, elem, node);
9070                 free(elem);
9071         }
9072 }
9073
9074 /*
9075  * Register array.
9076  */
9077 static struct regarray *
9078 regarray_find(struct rte_swx_pipeline *p, const char *name)
9079 {
9080         struct regarray *elem;
9081
9082         TAILQ_FOREACH(elem, &p->regarrays, node)
9083                 if (!strcmp(elem->name, name))
9084                         return elem;
9085
9086         return NULL;
9087 }
9088
9089 static struct regarray *
9090 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9091 {
9092         struct regarray *elem = NULL;
9093
9094         TAILQ_FOREACH(elem, &p->regarrays, node)
9095                 if (elem->id == id)
9096                         return elem;
9097
9098         return NULL;
9099 }
9100
9101 int
9102 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9103                               const char *name,
9104                               uint32_t size,
9105                               uint64_t init_val)
9106 {
9107         struct regarray *r;
9108
9109         CHECK(p, EINVAL);
9110
9111         CHECK_NAME(name, EINVAL);
9112         CHECK(!regarray_find(p, name), EEXIST);
9113
9114         CHECK(size, EINVAL);
9115         size = rte_align32pow2(size);
9116
9117         /* Memory allocation. */
9118         r = calloc(1, sizeof(struct regarray));
9119         CHECK(r, ENOMEM);
9120
9121         /* Node initialization. */
9122         strcpy(r->name, name);
9123         r->init_val = init_val;
9124         r->size = size;
9125         r->id = p->n_regarrays;
9126
9127         /* Node add to tailq. */
9128         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9129         p->n_regarrays++;
9130
9131         return 0;
9132 }
9133
9134 static int
9135 regarray_build(struct rte_swx_pipeline *p)
9136 {
9137         struct regarray *regarray;
9138
9139         if (!p->n_regarrays)
9140                 return 0;
9141
9142         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9143         CHECK(p->regarray_runtime, ENOMEM);
9144
9145         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9146                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9147                 uint32_t i;
9148
9149                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9150                                          RTE_CACHE_LINE_SIZE,
9151                                          p->numa_node);
9152                 CHECK(r->regarray, ENOMEM);
9153
9154                 if (regarray->init_val)
9155                         for (i = 0; i < regarray->size; i++)
9156                                 r->regarray[i] = regarray->init_val;
9157
9158                 r->size_mask = regarray->size - 1;
9159         }
9160
9161         return 0;
9162 }
9163
9164 static void
9165 regarray_build_free(struct rte_swx_pipeline *p)
9166 {
9167         uint32_t i;
9168
9169         if (!p->regarray_runtime)
9170                 return;
9171
9172         for (i = 0; i < p->n_regarrays; i++) {
9173                 struct regarray *regarray = regarray_find_by_id(p, i);
9174                 struct regarray_runtime *r = &p->regarray_runtime[i];
9175
9176                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9177         }
9178
9179         free(p->regarray_runtime);
9180         p->regarray_runtime = NULL;
9181 }
9182
9183 static void
9184 regarray_free(struct rte_swx_pipeline *p)
9185 {
9186         regarray_build_free(p);
9187
9188         for ( ; ; ) {
9189                 struct regarray *elem;
9190
9191                 elem = TAILQ_FIRST(&p->regarrays);
9192                 if (!elem)
9193                         break;
9194
9195                 TAILQ_REMOVE(&p->regarrays, elem, node);
9196                 free(elem);
9197         }
9198 }
9199
9200 /*
9201  * Meter array.
9202  */
9203 static struct meter_profile *
9204 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9205 {
9206         struct meter_profile *elem;
9207
9208         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9209                 if (!strcmp(elem->name, name))
9210                         return elem;
9211
9212         return NULL;
9213 }
9214
9215 static struct metarray *
9216 metarray_find(struct rte_swx_pipeline *p, const char *name)
9217 {
9218         struct metarray *elem;
9219
9220         TAILQ_FOREACH(elem, &p->metarrays, node)
9221                 if (!strcmp(elem->name, name))
9222                         return elem;
9223
9224         return NULL;
9225 }
9226
9227 static struct metarray *
9228 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9229 {
9230         struct metarray *elem = NULL;
9231
9232         TAILQ_FOREACH(elem, &p->metarrays, node)
9233                 if (elem->id == id)
9234                         return elem;
9235
9236         return NULL;
9237 }
9238
9239 int
9240 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9241                                  const char *name,
9242                                  uint32_t size)
9243 {
9244         struct metarray *m;
9245
9246         CHECK(p, EINVAL);
9247
9248         CHECK_NAME(name, EINVAL);
9249         CHECK(!metarray_find(p, name), EEXIST);
9250
9251         CHECK(size, EINVAL);
9252         size = rte_align32pow2(size);
9253
9254         /* Memory allocation. */
9255         m = calloc(1, sizeof(struct metarray));
9256         CHECK(m, ENOMEM);
9257
9258         /* Node initialization. */
9259         strcpy(m->name, name);
9260         m->size = size;
9261         m->id = p->n_metarrays;
9262
9263         /* Node add to tailq. */
9264         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9265         p->n_metarrays++;
9266
9267         return 0;
9268 }
9269
9270 struct meter_profile meter_profile_default = {
9271         .node = {0},
9272         .name = "",
9273         .params = {0},
9274
9275         .profile = {
9276                 .cbs = 10000,
9277                 .pbs = 10000,
9278                 .cir_period = 1,
9279                 .cir_bytes_per_period = 1,
9280                 .pir_period = 1,
9281                 .pir_bytes_per_period = 1,
9282         },
9283
9284         .n_users = 0,
9285 };
9286
9287 static void
9288 meter_init(struct meter *m)
9289 {
9290         memset(m, 0, sizeof(struct meter));
9291         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9292         m->profile = &meter_profile_default;
9293         m->color_mask = RTE_COLOR_GREEN;
9294
9295         meter_profile_default.n_users++;
9296 }
9297
9298 static int
9299 metarray_build(struct rte_swx_pipeline *p)
9300 {
9301         struct metarray *m;
9302
9303         if (!p->n_metarrays)
9304                 return 0;
9305
9306         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9307         CHECK(p->metarray_runtime, ENOMEM);
9308
9309         TAILQ_FOREACH(m, &p->metarrays, node) {
9310                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9311                 uint32_t i;
9312
9313                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9314                                          RTE_CACHE_LINE_SIZE,
9315                                          p->numa_node);
9316                 CHECK(r->metarray, ENOMEM);
9317
9318                 for (i = 0; i < m->size; i++)
9319                         meter_init(&r->metarray[i]);
9320
9321                 r->size_mask = m->size - 1;
9322         }
9323
9324         return 0;
9325 }
9326
9327 static void
9328 metarray_build_free(struct rte_swx_pipeline *p)
9329 {
9330         uint32_t i;
9331
9332         if (!p->metarray_runtime)
9333                 return;
9334
9335         for (i = 0; i < p->n_metarrays; i++) {
9336                 struct metarray *m = metarray_find_by_id(p, i);
9337                 struct metarray_runtime *r = &p->metarray_runtime[i];
9338
9339                 env_free(r->metarray, m->size * sizeof(struct meter));
9340         }
9341
9342         free(p->metarray_runtime);
9343         p->metarray_runtime = NULL;
9344 }
9345
9346 static void
9347 metarray_free(struct rte_swx_pipeline *p)
9348 {
9349         metarray_build_free(p);
9350
9351         /* Meter arrays. */
9352         for ( ; ; ) {
9353                 struct metarray *elem;
9354
9355                 elem = TAILQ_FIRST(&p->metarrays);
9356                 if (!elem)
9357                         break;
9358
9359                 TAILQ_REMOVE(&p->metarrays, elem, node);
9360                 free(elem);
9361         }
9362
9363         /* Meter profiles. */
9364         for ( ; ; ) {
9365                 struct meter_profile *elem;
9366
9367                 elem = TAILQ_FIRST(&p->meter_profiles);
9368                 if (!elem)
9369                         break;
9370
9371                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9372                 free(elem);
9373         }
9374 }
9375
9376 /*
9377  * Pipeline.
9378  */
9379 int
9380 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9381 {
9382         struct rte_swx_pipeline *pipeline;
9383
9384         /* Check input parameters. */
9385         CHECK(p, EINVAL);
9386
9387         /* Memory allocation. */
9388         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9389         CHECK(pipeline, ENOMEM);
9390
9391         /* Initialization. */
9392         TAILQ_INIT(&pipeline->struct_types);
9393         TAILQ_INIT(&pipeline->port_in_types);
9394         TAILQ_INIT(&pipeline->ports_in);
9395         TAILQ_INIT(&pipeline->port_out_types);
9396         TAILQ_INIT(&pipeline->ports_out);
9397         TAILQ_INIT(&pipeline->extern_types);
9398         TAILQ_INIT(&pipeline->extern_objs);
9399         TAILQ_INIT(&pipeline->extern_funcs);
9400         TAILQ_INIT(&pipeline->headers);
9401         TAILQ_INIT(&pipeline->actions);
9402         TAILQ_INIT(&pipeline->table_types);
9403         TAILQ_INIT(&pipeline->tables);
9404         TAILQ_INIT(&pipeline->regarrays);
9405         TAILQ_INIT(&pipeline->meter_profiles);
9406         TAILQ_INIT(&pipeline->metarrays);
9407
9408         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9409         pipeline->numa_node = numa_node;
9410
9411         *p = pipeline;
9412         return 0;
9413 }
9414
9415 void
9416 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9417 {
9418         if (!p)
9419                 return;
9420
9421         free(p->instructions);
9422
9423         metarray_free(p);
9424         regarray_free(p);
9425         table_state_free(p);
9426         table_free(p);
9427         action_free(p);
9428         metadata_free(p);
9429         header_free(p);
9430         extern_func_free(p);
9431         extern_obj_free(p);
9432         port_out_free(p);
9433         port_in_free(p);
9434         struct_free(p);
9435
9436         free(p);
9437 }
9438
9439 int
9440 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9441                                      const char **instructions,
9442                                      uint32_t n_instructions)
9443 {
9444         int err;
9445         uint32_t i;
9446
9447         err = instruction_config(p, NULL, instructions, n_instructions);
9448         if (err)
9449                 return err;
9450
9451         /* Thread instruction pointer reset. */
9452         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9453                 struct thread *t = &p->threads[i];
9454
9455                 thread_ip_reset(p, t);
9456         }
9457
9458         return 0;
9459 }
9460
9461 int
9462 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9463 {
9464         int status;
9465
9466         CHECK(p, EINVAL);
9467         CHECK(p->build_done == 0, EEXIST);
9468
9469         status = port_in_build(p);
9470         if (status)
9471                 goto error;
9472
9473         status = port_out_build(p);
9474         if (status)
9475                 goto error;
9476
9477         status = struct_build(p);
9478         if (status)
9479                 goto error;
9480
9481         status = extern_obj_build(p);
9482         if (status)
9483                 goto error;
9484
9485         status = extern_func_build(p);
9486         if (status)
9487                 goto error;
9488
9489         status = header_build(p);
9490         if (status)
9491                 goto error;
9492
9493         status = metadata_build(p);
9494         if (status)
9495                 goto error;
9496
9497         status = action_build(p);
9498         if (status)
9499                 goto error;
9500
9501         status = table_build(p);
9502         if (status)
9503                 goto error;
9504
9505         status = table_state_build(p);
9506         if (status)
9507                 goto error;
9508
9509         status = regarray_build(p);
9510         if (status)
9511                 goto error;
9512
9513         status = metarray_build(p);
9514         if (status)
9515                 goto error;
9516
9517         p->build_done = 1;
9518         return 0;
9519
9520 error:
9521         metarray_build_free(p);
9522         regarray_build_free(p);
9523         table_state_build_free(p);
9524         table_build_free(p);
9525         action_build_free(p);
9526         metadata_build_free(p);
9527         header_build_free(p);
9528         extern_func_build_free(p);
9529         extern_obj_build_free(p);
9530         port_out_build_free(p);
9531         port_in_build_free(p);
9532         struct_build_free(p);
9533
9534         return status;
9535 }
9536
9537 void
9538 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9539 {
9540         uint32_t i;
9541
9542         for (i = 0; i < n_instructions; i++)
9543                 instr_exec(p);
9544 }
9545
9546 void
9547 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9548 {
9549         uint32_t i;
9550
9551         for (i = 0; i < p->n_ports_out; i++) {
9552                 struct port_out_runtime *port = &p->out[i];
9553
9554                 if (port->flush)
9555                         port->flush(port->obj);
9556         }
9557 }
9558
9559 /*
9560  * Control.
9561  */
9562 int
9563 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9564                               struct rte_swx_ctl_pipeline_info *pipeline)
9565 {
9566         struct action *action;
9567         struct table *table;
9568         uint32_t n_actions = 0, n_tables = 0;
9569
9570         if (!p || !pipeline)
9571                 return -EINVAL;
9572
9573         TAILQ_FOREACH(action, &p->actions, node)
9574                 n_actions++;
9575
9576         TAILQ_FOREACH(table, &p->tables, node)
9577                 n_tables++;
9578
9579         pipeline->n_ports_in = p->n_ports_in;
9580         pipeline->n_ports_out = p->n_ports_out;
9581         pipeline->n_actions = n_actions;
9582         pipeline->n_tables = n_tables;
9583         pipeline->n_regarrays = p->n_regarrays;
9584         pipeline->n_metarrays = p->n_metarrays;
9585
9586         return 0;
9587 }
9588
9589 int
9590 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9591 {
9592         if (!p || !numa_node)
9593                 return -EINVAL;
9594
9595         *numa_node = p->numa_node;
9596         return 0;
9597 }
9598
9599 int
9600 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9601                             uint32_t action_id,
9602                             struct rte_swx_ctl_action_info *action)
9603 {
9604         struct action *a = NULL;
9605
9606         if (!p || (action_id >= p->n_actions) || !action)
9607                 return -EINVAL;
9608
9609         a = action_find_by_id(p, action_id);
9610         if (!a)
9611                 return -EINVAL;
9612
9613         strcpy(action->name, a->name);
9614         action->n_args = a->st ? a->st->n_fields : 0;
9615         return 0;
9616 }
9617
9618 int
9619 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9620                                 uint32_t action_id,
9621                                 uint32_t action_arg_id,
9622                                 struct rte_swx_ctl_action_arg_info *action_arg)
9623 {
9624         struct action *a = NULL;
9625         struct field *arg = NULL;
9626
9627         if (!p || (action_id >= p->n_actions) || !action_arg)
9628                 return -EINVAL;
9629
9630         a = action_find_by_id(p, action_id);
9631         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9632                 return -EINVAL;
9633
9634         arg = &a->st->fields[action_arg_id];
9635         strcpy(action_arg->name, arg->name);
9636         action_arg->n_bits = arg->n_bits;
9637
9638         return 0;
9639 }
9640
9641 int
9642 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9643                            uint32_t table_id,
9644                            struct rte_swx_ctl_table_info *table)
9645 {
9646         struct table *t = NULL;
9647
9648         if (!p || !table)
9649                 return -EINVAL;
9650
9651         t = table_find_by_id(p, table_id);
9652         if (!t)
9653                 return -EINVAL;
9654
9655         strcpy(table->name, t->name);
9656         strcpy(table->args, t->args);
9657         table->n_match_fields = t->n_fields;
9658         table->n_actions = t->n_actions;
9659         table->default_action_is_const = t->default_action_is_const;
9660         table->size = t->size;
9661         return 0;
9662 }
9663
9664 int
9665 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9666         uint32_t table_id,
9667         uint32_t match_field_id,
9668         struct rte_swx_ctl_table_match_field_info *match_field)
9669 {
9670         struct table *t;
9671         struct match_field *f;
9672
9673         if (!p || (table_id >= p->n_tables) || !match_field)
9674                 return -EINVAL;
9675
9676         t = table_find_by_id(p, table_id);
9677         if (!t || (match_field_id >= t->n_fields))
9678                 return -EINVAL;
9679
9680         f = &t->fields[match_field_id];
9681         match_field->match_type = f->match_type;
9682         match_field->is_header = t->header ? 1 : 0;
9683         match_field->n_bits = f->field->n_bits;
9684         match_field->offset = f->field->offset;
9685
9686         return 0;
9687 }
9688
9689 int
9690 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9691         uint32_t table_id,
9692         uint32_t table_action_id,
9693         struct rte_swx_ctl_table_action_info *table_action)
9694 {
9695         struct table *t;
9696
9697         if (!p || (table_id >= p->n_tables) || !table_action)
9698                 return -EINVAL;
9699
9700         t = table_find_by_id(p, table_id);
9701         if (!t || (table_action_id >= t->n_actions))
9702                 return -EINVAL;
9703
9704         table_action->action_id = t->actions[table_action_id]->id;
9705
9706         return 0;
9707 }
9708
9709 int
9710 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9711                           uint32_t table_id,
9712                           struct rte_swx_table_ops *table_ops,
9713                           int *is_stub)
9714 {
9715         struct table *t;
9716
9717         if (!p || (table_id >= p->n_tables))
9718                 return -EINVAL;
9719
9720         t = table_find_by_id(p, table_id);
9721         if (!t)
9722                 return -EINVAL;
9723
9724         if (t->type) {
9725                 if (table_ops)
9726                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9727                 *is_stub = 0;
9728         } else {
9729                 *is_stub = 1;
9730         }
9731
9732         return 0;
9733 }
9734
9735 int
9736 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9737                                  struct rte_swx_table_state **table_state)
9738 {
9739         if (!p || !table_state || !p->build_done)
9740                 return -EINVAL;
9741
9742         *table_state = p->table_state;
9743         return 0;
9744 }
9745
9746 int
9747 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9748                                  struct rte_swx_table_state *table_state)
9749 {
9750         if (!p || !table_state || !p->build_done)
9751                 return -EINVAL;
9752
9753         p->table_state = table_state;
9754         return 0;
9755 }
9756
9757 int
9758 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9759                                         uint32_t port_id,
9760                                         struct rte_swx_port_in_stats *stats)
9761 {
9762         struct port_in *port;
9763
9764         if (!p || !stats)
9765                 return -EINVAL;
9766
9767         port = port_in_find(p, port_id);
9768         if (!port)
9769                 return -EINVAL;
9770
9771         port->type->ops.stats_read(port->obj, stats);
9772         return 0;
9773 }
9774
9775 int
9776 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9777                                          uint32_t port_id,
9778                                          struct rte_swx_port_out_stats *stats)
9779 {
9780         struct port_out *port;
9781
9782         if (!p || !stats)
9783                 return -EINVAL;
9784
9785         port = port_out_find(p, port_id);
9786         if (!port)
9787                 return -EINVAL;
9788
9789         port->type->ops.stats_read(port->obj, stats);
9790         return 0;
9791 }
9792
9793 int
9794 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9795                                       const char *table_name,
9796                                       struct rte_swx_table_stats *stats)
9797 {
9798         struct table *table;
9799         struct table_statistics *table_stats;
9800
9801         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9802                 return -EINVAL;
9803
9804         table = table_find(p, table_name);
9805         if (!table)
9806                 return -EINVAL;
9807
9808         table_stats = &p->table_stats[table->id];
9809
9810         memcpy(&stats->n_pkts_action,
9811                &table_stats->n_pkts_action,
9812                p->n_actions * sizeof(uint64_t));
9813
9814         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9815         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9816
9817         return 0;
9818 }
9819
9820 int
9821 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9822                               uint32_t regarray_id,
9823                               struct rte_swx_ctl_regarray_info *regarray)
9824 {
9825         struct regarray *r;
9826
9827         if (!p || !regarray)
9828                 return -EINVAL;
9829
9830         r = regarray_find_by_id(p, regarray_id);
9831         if (!r)
9832                 return -EINVAL;
9833
9834         strcpy(regarray->name, r->name);
9835         regarray->size = r->size;
9836         return 0;
9837 }
9838
9839 int
9840 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9841                                    const char *regarray_name,
9842                                    uint32_t regarray_index,
9843                                    uint64_t *value)
9844 {
9845         struct regarray *regarray;
9846         struct regarray_runtime *r;
9847
9848         if (!p || !regarray_name || !value)
9849                 return -EINVAL;
9850
9851         regarray = regarray_find(p, regarray_name);
9852         if (!regarray || (regarray_index >= regarray->size))
9853                 return -EINVAL;
9854
9855         r = &p->regarray_runtime[regarray->id];
9856         *value = r->regarray[regarray_index];
9857         return 0;
9858 }
9859
9860 int
9861 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9862                                    const char *regarray_name,
9863                                    uint32_t regarray_index,
9864                                    uint64_t value)
9865 {
9866         struct regarray *regarray;
9867         struct regarray_runtime *r;
9868
9869         if (!p || !regarray_name)
9870                 return -EINVAL;
9871
9872         regarray = regarray_find(p, regarray_name);
9873         if (!regarray || (regarray_index >= regarray->size))
9874                 return -EINVAL;
9875
9876         r = &p->regarray_runtime[regarray->id];
9877         r->regarray[regarray_index] = value;
9878         return 0;
9879 }
9880
9881 int
9882 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9883                               uint32_t metarray_id,
9884                               struct rte_swx_ctl_metarray_info *metarray)
9885 {
9886         struct metarray *m;
9887
9888         if (!p || !metarray)
9889                 return -EINVAL;
9890
9891         m = metarray_find_by_id(p, metarray_id);
9892         if (!m)
9893                 return -EINVAL;
9894
9895         strcpy(metarray->name, m->name);
9896         metarray->size = m->size;
9897         return 0;
9898 }
9899
9900 int
9901 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9902                               const char *name,
9903                               struct rte_meter_trtcm_params *params)
9904 {
9905         struct meter_profile *mp;
9906         int status;
9907
9908         CHECK(p, EINVAL);
9909         CHECK_NAME(name, EINVAL);
9910         CHECK(params, EINVAL);
9911         CHECK(!meter_profile_find(p, name), EEXIST);
9912
9913         /* Node allocation. */
9914         mp = calloc(1, sizeof(struct meter_profile));
9915         CHECK(mp, ENOMEM);
9916
9917         /* Node initialization. */
9918         strcpy(mp->name, name);
9919         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9920         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9921         if (status) {
9922                 free(mp);
9923                 CHECK(0, EINVAL);
9924         }
9925
9926         /* Node add to tailq. */
9927         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9928
9929         return 0;
9930 }
9931
9932 int
9933 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9934                                  const char *name)
9935 {
9936         struct meter_profile *mp;
9937
9938         CHECK(p, EINVAL);
9939         CHECK_NAME(name, EINVAL);
9940
9941         mp = meter_profile_find(p, name);
9942         CHECK(mp, EINVAL);
9943         CHECK(!mp->n_users, EBUSY);
9944
9945         /* Remove node from tailq. */
9946         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9947         free(mp);
9948
9949         return 0;
9950 }
9951
9952 int
9953 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9954                         const char *metarray_name,
9955                         uint32_t metarray_index)
9956 {
9957         struct meter_profile *mp_old;
9958         struct metarray *metarray;
9959         struct metarray_runtime *metarray_runtime;
9960         struct meter *m;
9961
9962         CHECK(p, EINVAL);
9963         CHECK_NAME(metarray_name, EINVAL);
9964
9965         metarray = metarray_find(p, metarray_name);
9966         CHECK(metarray, EINVAL);
9967         CHECK(metarray_index < metarray->size, EINVAL);
9968
9969         metarray_runtime = &p->metarray_runtime[metarray->id];
9970         m = &metarray_runtime->metarray[metarray_index];
9971         mp_old = m->profile;
9972
9973         meter_init(m);
9974
9975         mp_old->n_users--;
9976
9977         return 0;
9978 }
9979
9980 int
9981 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9982                       const char *metarray_name,
9983                       uint32_t metarray_index,
9984                       const char *profile_name)
9985 {
9986         struct meter_profile *mp, *mp_old;
9987         struct metarray *metarray;
9988         struct metarray_runtime *metarray_runtime;
9989         struct meter *m;
9990
9991         CHECK(p, EINVAL);
9992         CHECK_NAME(metarray_name, EINVAL);
9993
9994         metarray = metarray_find(p, metarray_name);
9995         CHECK(metarray, EINVAL);
9996         CHECK(metarray_index < metarray->size, EINVAL);
9997
9998         mp = meter_profile_find(p, profile_name);
9999         CHECK(mp, EINVAL);
10000
10001         metarray_runtime = &p->metarray_runtime[metarray->id];
10002         m = &metarray_runtime->metarray[metarray_index];
10003         mp_old = m->profile;
10004
10005         memset(m, 0, sizeof(struct meter));
10006         rte_meter_trtcm_config(&m->m, &mp->profile);
10007         m->profile = mp;
10008         m->color_mask = RTE_COLORS;
10009
10010         mp->n_users++;
10011         mp_old->n_users--;
10012
10013         return 0;
10014 }
10015
10016 int
10017 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10018                              const char *metarray_name,
10019                              uint32_t metarray_index,
10020                              struct rte_swx_ctl_meter_stats *stats)
10021 {
10022         struct metarray *metarray;
10023         struct metarray_runtime *metarray_runtime;
10024         struct meter *m;
10025
10026         CHECK(p, EINVAL);
10027         CHECK_NAME(metarray_name, EINVAL);
10028
10029         metarray = metarray_find(p, metarray_name);
10030         CHECK(metarray, EINVAL);
10031         CHECK(metarray_index < metarray->size, EINVAL);
10032
10033         CHECK(stats, EINVAL);
10034
10035         metarray_runtime = &p->metarray_runtime[metarray->id];
10036         m = &metarray_runtime->metarray[metarray_index];
10037
10038         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10039         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10040
10041         return 0;
10042 }