lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <inttypes.h>
   9 #include <sys/queue.h>
  10 #include <arpa/inet.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_prefetch.h>
  14 #include <rte_byteorder.h>
  15 #include <rte_cycles.h>
  16 #include <rte_meter.h>
  17
  18 #include "rte_swx_pipeline.h"
  19 #include "rte_swx_ctl.h"
  20
  21 #define CHECK(condition, err_code)                                             \
  22 do {                                                                           \
  23         if (!(condition))                                                      \
  24                 return -(err_code);                                            \
  25 } while (0)
  26
  27 #define CHECK_NAME(name, err_code)                                             \
  28         CHECK((name) &&                                                        \
  29               (name)[0] &&                                                     \
  30               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  31               err_code)
  32
  33 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  34         CHECK((instr) &&                                                       \
  35               (instr)[0] &&                                                    \
  36               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  37                RTE_SWX_INSTRUCTION_SIZE),                                      \
  38               err_code)
  39
  40 #ifndef TRACE_LEVEL
  41 #define TRACE_LEVEL 0
  42 #endif
  43
  44 #if TRACE_LEVEL
  45 #define TRACE(...) printf(__VA_ARGS__)
  46 #else
  47 #define TRACE(...)
  48 #endif
  49
  50 /*
  51  * Environment.
  52  */
  53 #define ntoh64(x) rte_be_to_cpu_64(x)
  54 #define hton64(x) rte_cpu_to_be_64(x)
  55
  56 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  57
  58 #include <rte_malloc.h>
  59
  60 static void *
  61 env_malloc(size_t size, size_t alignment, int numa_node)
  62 {
  63         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  64 }
  65
  66 static void
  67 env_free(void *start, size_t size __rte_unused)
  68 {
  69         rte_free(start);
  70 }
  71
  72 #else
  73
  74 #include <numa.h>
  75
  76 static void *
  77 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  78 {
  79         void *start;
  80
  81         if (numa_available() == -1)
  82                 return NULL;
  83
  84         start = numa_alloc_onnode(size, numa_node);
  85         if (!start)
  86                 return NULL;
  87
  88         memset(start, 0, size);
  89         return start;
  90 }
  91
  92 static void
  93 env_free(void *start, size_t size)
  94 {
  95         if (numa_available() == -1)
  96                 return;
  97
  98         numa_free(start, size);
  99 }
 100
 101 #endif
 102
 103 /*
 104  * Struct.
 105  */
 106 struct field {
 107         char name[RTE_SWX_NAME_SIZE];
 108         uint32_t n_bits;
 109         uint32_t offset;
 110 };
 111
 112 struct struct_type {
 113         TAILQ_ENTRY(struct_type) node;
 114         char name[RTE_SWX_NAME_SIZE];
 115         struct field *fields;
 116         uint32_t n_fields;
 117         uint32_t n_bits;
 118 };
 119
 120 TAILQ_HEAD(struct_type_tailq, struct_type);
 121
 122 /*
 123  * Input port.
 124  */
 125 struct port_in_type {
 126         TAILQ_ENTRY(port_in_type) node;
 127         char name[RTE_SWX_NAME_SIZE];
 128         struct rte_swx_port_in_ops ops;
 129 };
 130
 131 TAILQ_HEAD(port_in_type_tailq, port_in_type);
 132
 133 struct port_in {
 134         TAILQ_ENTRY(port_in) node;
 135         struct port_in_type *type;
 136         void *obj;
 137         uint32_t id;
 138 };
 139
 140 TAILQ_HEAD(port_in_tailq, port_in);
 141
 142 struct port_in_runtime {
 143         rte_swx_port_in_pkt_rx_t pkt_rx;
 144         void *obj;
 145 };
 146
 147 /*
 148  * Output port.
 149  */
 150 struct port_out_type {
 151         TAILQ_ENTRY(port_out_type) node;
 152         char name[RTE_SWX_NAME_SIZE];
 153         struct rte_swx_port_out_ops ops;
 154 };
 155
 156 TAILQ_HEAD(port_out_type_tailq, port_out_type);
 157
 158 struct port_out {
 159         TAILQ_ENTRY(port_out) node;
 160         struct port_out_type *type;
 161         void *obj;
 162         uint32_t id;
 163 };
 164
 165 TAILQ_HEAD(port_out_tailq, port_out);
 166
 167 struct port_out_runtime {
 168         rte_swx_port_out_pkt_tx_t pkt_tx;
 169         rte_swx_port_out_flush_t flush;
 170         void *obj;
 171 };
 172
 173 /*
 174  * Extern object.
 175  */
 176 struct extern_type_member_func {
 177         TAILQ_ENTRY(extern_type_member_func) node;
 178         char name[RTE_SWX_NAME_SIZE];
 179         rte_swx_extern_type_member_func_t func;
 180         uint32_t id;
 181 };
 182
 183 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 184
 185 struct extern_type {
 186         TAILQ_ENTRY(extern_type) node;
 187         char name[RTE_SWX_NAME_SIZE];
 188         struct struct_type *mailbox_struct_type;
 189         rte_swx_extern_type_constructor_t constructor;
 190         rte_swx_extern_type_destructor_t destructor;
 191         struct extern_type_member_func_tailq funcs;
 192         uint32_t n_funcs;
 193 };
 194
 195 TAILQ_HEAD(extern_type_tailq, extern_type);
 196
 197 struct extern_obj {
 198         TAILQ_ENTRY(extern_obj) node;
 199         char name[RTE_SWX_NAME_SIZE];
 200         struct extern_type *type;
 201         void *obj;
 202         uint32_t struct_id;
 203         uint32_t id;
 204 };
 205
 206 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 207
 208 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 209 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 210 #endif
 211
 212 struct extern_obj_runtime {
 213         void *obj;
 214         uint8_t *mailbox;
 215         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 216 };
 217
 218 /*
 219  * Extern function.
 220  */
 221 struct extern_func {
 222         TAILQ_ENTRY(extern_func) node;
 223         char name[RTE_SWX_NAME_SIZE];
 224         struct struct_type *mailbox_struct_type;
 225         rte_swx_extern_func_t func;
 226         uint32_t struct_id;
 227         uint32_t id;
 228 };
 229
 230 TAILQ_HEAD(extern_func_tailq, extern_func);
 231
 232 struct extern_func_runtime {
 233         uint8_t *mailbox;
 234         rte_swx_extern_func_t func;
 235 };
 236
 237 /*
 238  * Header.
 239  */
 240 struct header {
 241         TAILQ_ENTRY(header) node;
 242         char name[RTE_SWX_NAME_SIZE];
 243         struct struct_type *st;
 244         uint32_t struct_id;
 245         uint32_t id;
 246 };
 247
 248 TAILQ_HEAD(header_tailq, header);
 249
 250 struct header_runtime {
 251         uint8_t *ptr0;
 252 };
 253
 254 struct header_out_runtime {
 255         uint8_t *ptr0;
 256         uint8_t *ptr;
 257         uint32_t n_bytes;
 258 };
 259
 260 /*
 261  * Instruction.
 262  */
 263
 264 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 265  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 266  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 267  * when transferred to packet meta-data and in NBO when transferred to packet
 268  * headers.
 269  */
 270
 271 /* Notation conventions:
 272  *    -Header field: H = h.header.field (dst/src)
 273  *    -Meta-data field: M = m.field (dst/src)
 274  *    -Extern object mailbox field: E = e.field (dst/src)
 275  *    -Extern function mailbox field: F = f.field (dst/src)
 276  *    -Table action data field: T = t.field (src only)
 277  *    -Immediate value: I = 32-bit unsigned value (src only)
 278  */
 279
 280 enum instruction_type {
 281         /* rx m.port_in */
 282         INSTR_RX,
 283
 284         /* tx port_out
 285          * port_out = MI
 286          */
 287         INSTR_TX,   /* port_out = M */
 288         INSTR_TX_I, /* port_out = I */
 289
 290         /* extract h.header */
 291         INSTR_HDR_EXTRACT,
 292         INSTR_HDR_EXTRACT2,
 293         INSTR_HDR_EXTRACT3,
 294         INSTR_HDR_EXTRACT4,
 295         INSTR_HDR_EXTRACT5,
 296         INSTR_HDR_EXTRACT6,
 297         INSTR_HDR_EXTRACT7,
 298         INSTR_HDR_EXTRACT8,
 299
 300         /* emit h.header */
 301         INSTR_HDR_EMIT,
 302         INSTR_HDR_EMIT_TX,
 303         INSTR_HDR_EMIT2_TX,
 304         INSTR_HDR_EMIT3_TX,
 305         INSTR_HDR_EMIT4_TX,
 306         INSTR_HDR_EMIT5_TX,
 307         INSTR_HDR_EMIT6_TX,
 308         INSTR_HDR_EMIT7_TX,
 309         INSTR_HDR_EMIT8_TX,
 310
 311         /* validate h.header */
 312         INSTR_HDR_VALIDATE,
 313
 314         /* invalidate h.header */
 315         INSTR_HDR_INVALIDATE,
 316
 317         /* mov dst src
 318          * dst = src
 319          * dst = HMEF, src = HMEFTI
 320          */
 321         INSTR_MOV,   /* dst = MEF, src = MEFT */
 322         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 323         INSTR_MOV_I, /* dst = HMEF, src = I */
 324
 325         /* dma h.header t.field
 326          * memcpy(h.header, t.field, sizeof(h.header))
 327          */
 328         INSTR_DMA_HT,
 329         INSTR_DMA_HT2,
 330         INSTR_DMA_HT3,
 331         INSTR_DMA_HT4,
 332         INSTR_DMA_HT5,
 333         INSTR_DMA_HT6,
 334         INSTR_DMA_HT7,
 335         INSTR_DMA_HT8,
 336
 337         /* add dst src
 338          * dst += src
 339          * dst = HMEF, src = HMEFTI
 340          */
 341         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 342         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 343         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 344         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 345         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 346         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 347
 348         /* sub dst src
 349          * dst -= src
 350          * dst = HMEF, src = HMEFTI
 351          */
 352         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 353         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 354         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 355         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 356         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 357         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 358
 359         /* ckadd dst src
 360          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 361          * dst = H, src = {H, h.header}
 362          */
 363         INSTR_ALU_CKADD_FIELD,    /* src = H */
 364         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 365         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 366
 367         /* cksub dst src
 368          * dst = dst '- src
 369          * dst = H, src = H
 370          */
 371         INSTR_ALU_CKSUB_FIELD,
 372
 373         /* and dst src
 374          * dst &= src
 375          * dst = HMEF, src = HMEFTI
 376          */
 377         INSTR_ALU_AND,   /* dst = MEF, src = MEFT */
 378         INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 379         INSTR_ALU_AND_I, /* dst = HMEF, src = I */
 380
 381         /* or dst src
 382          * dst |= src
 383          * dst = HMEF, src = HMEFTI
 384          */
 385         INSTR_ALU_OR,   /* dst = MEF, src = MEFT */
 386         INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 387         INSTR_ALU_OR_I, /* dst = HMEF, src = I */
 388
 389         /* xor dst src
 390          * dst ^= src
 391          * dst = HMEF, src = HMEFTI
 392          */
 393         INSTR_ALU_XOR,   /* dst = MEF, src = MEFT */
 394         INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 395         INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
 396
 397         /* shl dst src
 398          * dst <<= src
 399          * dst = HMEF, src = HMEFTI
 400          */
 401         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 402         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 403         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 404         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 405         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 406         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 407
 408         /* shr dst src
 409          * dst >>= src
 410          * dst = HMEF, src = HMEFTI
 411          */
 412         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 413         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 414         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 415         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 416         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 417         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 418
 419         /* regprefetch REGARRAY index
 420          * prefetch REGARRAY[index]
 421          * index = HMEFTI
 422          */
 423         INSTR_REGPREFETCH_RH, /* index = H */
 424         INSTR_REGPREFETCH_RM, /* index = MEFT */
 425         INSTR_REGPREFETCH_RI, /* index = I */
 426
 427         /* regrd dst REGARRAY index
 428          * dst = REGARRAY[index]
 429          * dst = HMEF, index = HMEFTI
 430          */
 431         INSTR_REGRD_HRH, /* dst = H, index = H */
 432         INSTR_REGRD_HRM, /* dst = H, index = MEFT */
 433         INSTR_REGRD_HRI, /* dst = H, index = I */
 434         INSTR_REGRD_MRH, /* dst = MEF, index = H */
 435         INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
 436         INSTR_REGRD_MRI, /* dst = MEF, index = I */
 437
 438         /* regwr REGARRAY index src
 439          * REGARRAY[index] = src
 440          * index = HMEFTI, src = HMEFTI
 441          */
 442         INSTR_REGWR_RHH, /* index = H, src = H */
 443         INSTR_REGWR_RHM, /* index = H, src = MEFT */
 444         INSTR_REGWR_RHI, /* index = H, src = I */
 445         INSTR_REGWR_RMH, /* index = MEFT, src = H */
 446         INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
 447         INSTR_REGWR_RMI, /* index = MEFT, src = I */
 448         INSTR_REGWR_RIH, /* index = I, src = H */
 449         INSTR_REGWR_RIM, /* index = I, src = MEFT */
 450         INSTR_REGWR_RII, /* index = I, src = I */
 451
 452         /* regadd REGARRAY index src
 453          * REGARRAY[index] += src
 454          * index = HMEFTI, src = HMEFTI
 455          */
 456         INSTR_REGADD_RHH, /* index = H, src = H */
 457         INSTR_REGADD_RHM, /* index = H, src = MEFT */
 458         INSTR_REGADD_RHI, /* index = H, src = I */
 459         INSTR_REGADD_RMH, /* index = MEFT, src = H */
 460         INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
 461         INSTR_REGADD_RMI, /* index = MEFT, src = I */
 462         INSTR_REGADD_RIH, /* index = I, src = H */
 463         INSTR_REGADD_RIM, /* index = I, src = MEFT */
 464         INSTR_REGADD_RII, /* index = I, src = I */
 465
 466         /* metprefetch METARRAY index
 467          * prefetch METARRAY[index]
 468          * index = HMEFTI
 469          */
 470         INSTR_METPREFETCH_H, /* index = H */
 471         INSTR_METPREFETCH_M, /* index = MEFT */
 472         INSTR_METPREFETCH_I, /* index = I */
 473
 474         /* meter METARRAY index length color_in color_out
 475          * color_out = meter(METARRAY[index], length, color_in)
 476          * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
 477          */
 478         INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
 479         INSTR_METER_HHI, /* index = H, length = H, color_in = I */
 480         INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
 481         INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
 482         INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
 483         INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
 484         INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
 485         INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
 486         INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
 487         INSTR_METER_IHI, /* index = I, length = H, color_in = I */
 488         INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
 489         INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
 490
 491         /* table TABLE */
 492         INSTR_TABLE,
 493
 494         /* extern e.obj.func */
 495         INSTR_EXTERN_OBJ,
 496
 497         /* extern f.func */
 498         INSTR_EXTERN_FUNC,
 499
 500         /* jmp LABEL
 501          * Unconditional jump
 502          */
 503         INSTR_JMP,
 504
 505         /* jmpv LABEL h.header
 506          * Jump if header is valid
 507          */
 508         INSTR_JMP_VALID,
 509
 510         /* jmpnv LABEL h.header
 511          * Jump if header is invalid
 512          */
 513         INSTR_JMP_INVALID,
 514
 515         /* jmph LABEL
 516          * Jump if table lookup hit
 517          */
 518         INSTR_JMP_HIT,
 519
 520         /* jmpnh LABEL
 521          * Jump if table lookup miss
 522          */
 523         INSTR_JMP_MISS,
 524
 525         /* jmpa LABEL ACTION
 526          * Jump if action run
 527          */
 528         INSTR_JMP_ACTION_HIT,
 529
 530         /* jmpna LABEL ACTION
 531          * Jump if action not run
 532          */
 533         INSTR_JMP_ACTION_MISS,
 534
 535         /* jmpeq LABEL a b
 536          * Jump is a is equal to b
 537          * a = HMEFT, b = HMEFTI
 538          */
 539         INSTR_JMP_EQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
 540         INSTR_JMP_EQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
 541         INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 542
 543         /* jmpneq LABEL a b
 544          * Jump is a is not equal to b
 545          * a = HMEFT, b = HMEFTI
 546          */
 547         INSTR_JMP_NEQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
 548         INSTR_JMP_NEQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
 549         INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 550
 551         /* jmplt LABEL a b
 552          * Jump if a is less than b
 553          * a = HMEFT, b = HMEFTI
 554          */
 555         INSTR_JMP_LT,    /* a = MEF, b = MEF */
 556         INSTR_JMP_LT_MH, /* a = MEF, b = H */
 557         INSTR_JMP_LT_HM, /* a = H, b = MEF */
 558         INSTR_JMP_LT_HH, /* a = H, b = H */
 559         INSTR_JMP_LT_MI, /* a = MEF, b = I */
 560         INSTR_JMP_LT_HI, /* a = H, b = I */
 561
 562         /* jmpgt LABEL a b
 563          * Jump if a is greater than b
 564          * a = HMEFT, b = HMEFTI
 565          */
 566         INSTR_JMP_GT,    /* a = MEF, b = MEF */
 567         INSTR_JMP_GT_MH, /* a = MEF, b = H */
 568         INSTR_JMP_GT_HM, /* a = H, b = MEF */
 569         INSTR_JMP_GT_HH, /* a = H, b = H */
 570         INSTR_JMP_GT_MI, /* a = MEF, b = I */
 571         INSTR_JMP_GT_HI, /* a = H, b = I */
 572
 573         /* return
 574          * Return from action
 575          */
 576         INSTR_RETURN,
 577 };
 578
 579 struct instr_operand {
 580         uint8_t struct_id;
 581         uint8_t n_bits;
 582         uint8_t offset;
 583         uint8_t pad;
 584 };
 585
 586 struct instr_io {
 587         struct {
 588                 union {
 589                         struct {
 590                                 uint8_t offset;
 591                                 uint8_t n_bits;
 592                                 uint8_t pad[2];
 593                         };
 594
 595                         uint32_t val;
 596                 };
 597         } io;
 598
 599         struct {
 600                 uint8_t header_id[8];
 601                 uint8_t struct_id[8];
 602                 uint8_t n_bytes[8];
 603         } hdr;
 604 };
 605
 606 struct instr_hdr_validity {
 607         uint8_t header_id;
 608 };
 609
 610 struct instr_table {
 611         uint8_t table_id;
 612 };
 613
 614 struct instr_extern_obj {
 615         uint8_t ext_obj_id;
 616         uint8_t func_id;
 617 };
 618
 619 struct instr_extern_func {
 620         uint8_t ext_func_id;
 621 };
 622
 623 struct instr_dst_src {
 624         struct instr_operand dst;
 625         union {
 626                 struct instr_operand src;
 627                 uint64_t src_val;
 628         };
 629 };
 630
 631 struct instr_regarray {
 632         uint8_t regarray_id;
 633         uint8_t pad[3];
 634
 635         union {
 636                 struct instr_operand idx;
 637                 uint32_t idx_val;
 638         };
 639
 640         union {
 641                 struct instr_operand dstsrc;
 642                 uint64_t dstsrc_val;
 643         };
 644 };
 645
 646 struct instr_meter {
 647         uint8_t metarray_id;
 648         uint8_t pad[3];
 649
 650         union {
 651                 struct instr_operand idx;
 652                 uint32_t idx_val;
 653         };
 654
 655         struct instr_operand length;
 656
 657         union {
 658                 struct instr_operand color_in;
 659                 uint32_t color_in_val;
 660         };
 661
 662         struct instr_operand color_out;
 663 };
 664
 665 struct instr_dma {
 666         struct {
 667                 uint8_t header_id[8];
 668                 uint8_t struct_id[8];
 669         } dst;
 670
 671         struct {
 672                 uint8_t offset[8];
 673         } src;
 674
 675         uint16_t n_bytes[8];
 676 };
 677
 678 struct instr_jmp {
 679         struct instruction *ip;
 680
 681         union {
 682                 struct instr_operand a;
 683                 uint8_t header_id;
 684                 uint8_t action_id;
 685         };
 686
 687         union {
 688                 struct instr_operand b;
 689                 uint64_t b_val;
 690         };
 691 };
 692
 693 struct instruction {
 694         enum instruction_type type;
 695         union {
 696                 struct instr_io io;
 697                 struct instr_hdr_validity valid;
 698                 struct instr_dst_src mov;
 699                 struct instr_regarray regarray;
 700                 struct instr_meter meter;
 701                 struct instr_dma dma;
 702                 struct instr_dst_src alu;
 703                 struct instr_table table;
 704                 struct instr_extern_obj ext_obj;
 705                 struct instr_extern_func ext_func;
 706                 struct instr_jmp jmp;
 707         };
 708 };
 709
 710 struct instruction_data {
 711         char label[RTE_SWX_NAME_SIZE];
 712         char jmp_label[RTE_SWX_NAME_SIZE];
 713         uint32_t n_users; /* user = jmp instruction to this instruction. */
 714         int invalid;
 715 };
 716
 717 /*
 718  * Action.
 719  */
 720 struct action {
 721         TAILQ_ENTRY(action) node;
 722         char name[RTE_SWX_NAME_SIZE];
 723         struct struct_type *st;
 724         struct instruction *instructions;
 725         uint32_t n_instructions;
 726         uint32_t id;
 727 };
 728
 729 TAILQ_HEAD(action_tailq, action);
 730
 731 /*
 732  * Table.
 733  */
 734 struct table_type {
 735         TAILQ_ENTRY(table_type) node;
 736         char name[RTE_SWX_NAME_SIZE];
 737         enum rte_swx_table_match_type match_type;
 738         struct rte_swx_table_ops ops;
 739 };
 740
 741 TAILQ_HEAD(table_type_tailq, table_type);
 742
 743 struct match_field {
 744         enum rte_swx_table_match_type match_type;
 745         struct field *field;
 746 };
 747
 748 struct table {
 749         TAILQ_ENTRY(table) node;
 750         char name[RTE_SWX_NAME_SIZE];
 751         char args[RTE_SWX_NAME_SIZE];
 752         struct table_type *type; /* NULL when n_fields == 0. */
 753
 754         /* Match. */
 755         struct match_field *fields;
 756         uint32_t n_fields;
 757         struct header *header; /* Only valid when n_fields > 0. */
 758
 759         /* Action. */
 760         struct action **actions;
 761         struct action *default_action;
 762         uint8_t *default_action_data;
 763         uint32_t n_actions;
 764         int default_action_is_const;
 765         uint32_t action_data_size_max;
 766
 767         uint32_t size;
 768         uint32_t id;
 769 };
 770
 771 TAILQ_HEAD(table_tailq, table);
 772
 773 struct table_runtime {
 774         rte_swx_table_lookup_t func;
 775         void *mailbox;
 776         uint8_t **key;
 777 };
 778
 779 struct table_statistics {
 780         uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
 781         uint64_t *n_pkts_action;
 782 };
 783
 784 /*
 785  * Register array.
 786  */
 787 struct regarray {
 788         TAILQ_ENTRY(regarray) node;
 789         char name[RTE_SWX_NAME_SIZE];
 790         uint64_t init_val;
 791         uint32_t size;
 792         uint32_t id;
 793 };
 794
 795 TAILQ_HEAD(regarray_tailq, regarray);
 796
 797 struct regarray_runtime {
 798         uint64_t *regarray;
 799         uint32_t size_mask;
 800 };
 801
 802 /*
 803  * Meter array.
 804  */
 805 struct meter_profile {
 806         TAILQ_ENTRY(meter_profile) node;
 807         char name[RTE_SWX_NAME_SIZE];
 808         struct rte_meter_trtcm_params params;
 809         struct rte_meter_trtcm_profile profile;
 810         uint32_t n_users;
 811 };
 812
 813 TAILQ_HEAD(meter_profile_tailq, meter_profile);
 814
 815 struct metarray {
 816         TAILQ_ENTRY(metarray) node;
 817         char name[RTE_SWX_NAME_SIZE];
 818         uint32_t size;
 819         uint32_t id;
 820 };
 821
 822 TAILQ_HEAD(metarray_tailq, metarray);
 823
 824 struct meter {
 825         struct rte_meter_trtcm m;
 826         struct meter_profile *profile;
 827         enum rte_color color_mask;
 828         uint8_t pad[20];
 829
 830         uint64_t n_pkts[RTE_COLORS];
 831         uint64_t n_bytes[RTE_COLORS];
 832 };
 833
 834 struct metarray_runtime {
 835         struct meter *metarray;
 836         uint32_t size_mask;
 837 };
 838
 839 /*
 840  * Pipeline.
 841  */
 842 struct thread {
 843         /* Packet. */
 844         struct rte_swx_pkt pkt;
 845         uint8_t *ptr;
 846
 847         /* Structures. */
 848         uint8_t **structs;
 849
 850         /* Packet headers. */
 851         struct header_runtime *headers; /* Extracted or generated headers. */
 852         struct header_out_runtime *headers_out; /* Emitted headers. */
 853         uint8_t *header_storage;
 854         uint8_t *header_out_storage;
 855         uint64_t valid_headers;
 856         uint32_t n_headers_out;
 857
 858         /* Packet meta-data. */
 859         uint8_t *metadata;
 860
 861         /* Tables. */
 862         struct table_runtime *tables;
 863         struct rte_swx_table_state *table_state;
 864         uint64_t action_id;
 865         int hit; /* 0 = Miss, 1 = Hit. */
 866
 867         /* Extern objects and functions. */
 868         struct extern_obj_runtime *extern_objs;
 869         struct extern_func_runtime *extern_funcs;
 870
 871         /* Instructions. */
 872         struct instruction *ip;
 873         struct instruction *ret;
 874 };
 875
 876 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 877 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 878 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 879
 880 #define HEADER_VALID(thread, header_id) \
 881         MASK64_BIT_GET((thread)->valid_headers, header_id)
 882
 883 #define ALU(thread, ip, operator)  \
 884 {                                                                              \
 885         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 886         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 887         uint64_t dst64 = *dst64_ptr;                                           \
 888         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 889         uint64_t dst = dst64 & dst64_mask;                                     \
 890                                                                                \
 891         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 892         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 893         uint64_t src64 = *src64_ptr;                                           \
 894         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 895         uint64_t src = src64 & src64_mask;                                     \
 896                                                                                \
 897         uint64_t result = dst operator src;                                    \
 898                                                                                \
 899         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 900 }
 901
 902 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 903
 904 #define ALU_S(thread, ip, operator)  \
 905 {                                                                              \
 906         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 907         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 908         uint64_t dst64 = *dst64_ptr;                                           \
 909         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 910         uint64_t dst = dst64 & dst64_mask;                                     \
 911                                                                                \
 912         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 913         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 914         uint64_t src64 = *src64_ptr;                                           \
 915         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 916                                                                                \
 917         uint64_t result = dst operator src;                                    \
 918                                                                                \
 919         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 920 }
 921
 922 #define ALU_MH ALU_S
 923
 924 #define ALU_HM(thread, ip, operator)  \
 925 {                                                                              \
 926         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 927         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 928         uint64_t dst64 = *dst64_ptr;                                           \
 929         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 930         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 931                                                                                \
 932         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 933         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 934         uint64_t src64 = *src64_ptr;                                           \
 935         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 936         uint64_t src = src64 & src64_mask;                                     \
 937                                                                                \
 938         uint64_t result = dst operator src;                                    \
 939         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 940                                                                                \
 941         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 942 }
 943
 944 #define ALU_HH(thread, ip, operator)  \
 945 {                                                                              \
 946         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 947         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 948         uint64_t dst64 = *dst64_ptr;                                           \
 949         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 950         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 951                                                                                \
 952         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 953         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 954         uint64_t src64 = *src64_ptr;                                           \
 955         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 956                                                                                \
 957         uint64_t result = dst operator src;                                    \
 958         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 959                                                                                \
 960         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 961 }
 962
 963 #else
 964
 965 #define ALU_S ALU
 966 #define ALU_MH ALU
 967 #define ALU_HM ALU
 968 #define ALU_HH ALU
 969
 970 #endif
 971
 972 #define ALU_I(thread, ip, operator)  \
 973 {                                                                              \
 974         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 975         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 976         uint64_t dst64 = *dst64_ptr;                                           \
 977         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 978         uint64_t dst = dst64 & dst64_mask;                                     \
 979                                                                                \
 980         uint64_t src = (ip)->alu.src_val;                                      \
 981                                                                                \
 982         uint64_t result = dst operator src;                                    \
 983                                                                                \
 984         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 985 }
 986
 987 #define ALU_MI ALU_I
 988
 989 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 990
 991 #define ALU_HI(thread, ip, operator)  \
 992 {                                                                              \
 993         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 994         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 995         uint64_t dst64 = *dst64_ptr;                                           \
 996         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 997         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 998                                                                                \
 999         uint64_t src = (ip)->alu.src_val;                                      \
1000                                                                                \
1001         uint64_t result = dst operator src;                                    \
1002         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1003                                                                                \
1004         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1005 }
1006
1007 #else
1008
1009 #define ALU_HI ALU_I
1010
1011 #endif
1012
1013 #define MOV(thread, ip)  \
1014 {                                                                              \
1015         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1016         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1017         uint64_t dst64 = *dst64_ptr;                                           \
1018         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1019                                                                                \
1020         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1021         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1022         uint64_t src64 = *src64_ptr;                                           \
1023         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1024         uint64_t src = src64 & src64_mask;                                     \
1025                                                                                \
1026         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1027 }
1028
1029 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1030
1031 #define MOV_S(thread, ip)  \
1032 {                                                                              \
1033         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1034         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1035         uint64_t dst64 = *dst64_ptr;                                           \
1036         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1037                                                                                \
1038         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1039         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1040         uint64_t src64 = *src64_ptr;                                           \
1041         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
1042                                                                                \
1043         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1044 }
1045
1046 #else
1047
1048 #define MOV_S MOV
1049
1050 #endif
1051
1052 #define MOV_I(thread, ip)  \
1053 {                                                                              \
1054         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1055         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1056         uint64_t dst64 = *dst64_ptr;                                           \
1057         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1058                                                                                \
1059         uint64_t src = (ip)->mov.src_val;                                      \
1060                                                                                \
1061         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1062 }
1063
1064 #define JMP_CMP(thread, ip, operator)  \
1065 {                                                                              \
1066         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1067         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1068         uint64_t a64 = *a64_ptr;                                               \
1069         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1070         uint64_t a = a64 & a64_mask;                                           \
1071                                                                                \
1072         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1073         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1074         uint64_t b64 = *b64_ptr;                                               \
1075         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1076         uint64_t b = b64 & b64_mask;                                           \
1077                                                                                \
1078         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1079 }
1080
1081 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1082
1083 #define JMP_CMP_S(thread, ip, operator)  \
1084 {                                                                              \
1085         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1086         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1087         uint64_t a64 = *a64_ptr;                                               \
1088         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1089         uint64_t a = a64 & a64_mask;                                           \
1090                                                                                \
1091         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1092         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1093         uint64_t b64 = *b64_ptr;                                               \
1094         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1095                                                                                \
1096         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1097 }
1098
1099 #define JMP_CMP_MH JMP_CMP_S
1100
1101 #define JMP_CMP_HM(thread, ip, operator)  \
1102 {                                                                              \
1103         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1104         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1105         uint64_t a64 = *a64_ptr;                                               \
1106         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1107                                                                                \
1108         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1109         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1110         uint64_t b64 = *b64_ptr;                                               \
1111         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1112         uint64_t b = b64 & b64_mask;                                           \
1113                                                                                \
1114         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1115 }
1116
1117 #define JMP_CMP_HH(thread, ip, operator)  \
1118 {                                                                              \
1119         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1120         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1121         uint64_t a64 = *a64_ptr;                                               \
1122         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1123                                                                                \
1124         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1125         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1126         uint64_t b64 = *b64_ptr;                                               \
1127         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1128                                                                                \
1129         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1130 }
1131
1132 #else
1133
1134 #define JMP_CMP_S JMP_CMP
1135 #define JMP_CMP_MH JMP_CMP
1136 #define JMP_CMP_HM JMP_CMP
1137 #define JMP_CMP_HH JMP_CMP
1138
1139 #endif
1140
1141 #define JMP_CMP_I(thread, ip, operator)  \
1142 {                                                                              \
1143         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1144         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1145         uint64_t a64 = *a64_ptr;                                               \
1146         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1147         uint64_t a = a64 & a64_mask;                                           \
1148                                                                                \
1149         uint64_t b = (ip)->jmp.b_val;                                          \
1150                                                                                \
1151         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1152 }
1153
1154 #define JMP_CMP_MI JMP_CMP_I
1155
1156 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1157
1158 #define JMP_CMP_HI(thread, ip, operator)  \
1159 {                                                                              \
1160         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1161         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1162         uint64_t a64 = *a64_ptr;                                               \
1163         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1164                                                                                \
1165         uint64_t b = (ip)->jmp.b_val;                                          \
1166                                                                                \
1167         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1168 }
1169
1170 #else
1171
1172 #define JMP_CMP_HI JMP_CMP_I
1173
1174 #endif
1175
1176 #define METADATA_READ(thread, offset, n_bits)                                  \
1177 ({                                                                             \
1178         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1179         uint64_t m64 = *m64_ptr;                                               \
1180         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1181         (m64 & m64_mask);                                                      \
1182 })
1183
1184 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
1185 {                                                                              \
1186         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1187         uint64_t m64 = *m64_ptr;                                               \
1188         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1189                                                                                \
1190         uint64_t m_new = value;                                                \
1191                                                                                \
1192         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
1193 }
1194
1195 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1196 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1197 #endif
1198
1199 struct rte_swx_pipeline {
1200         struct struct_type_tailq struct_types;
1201         struct port_in_type_tailq port_in_types;
1202         struct port_in_tailq ports_in;
1203         struct port_out_type_tailq port_out_types;
1204         struct port_out_tailq ports_out;
1205         struct extern_type_tailq extern_types;
1206         struct extern_obj_tailq extern_objs;
1207         struct extern_func_tailq extern_funcs;
1208         struct header_tailq headers;
1209         struct struct_type *metadata_st;
1210         uint32_t metadata_struct_id;
1211         struct action_tailq actions;
1212         struct table_type_tailq table_types;
1213         struct table_tailq tables;
1214         struct regarray_tailq regarrays;
1215         struct meter_profile_tailq meter_profiles;
1216         struct metarray_tailq metarrays;
1217
1218         struct port_in_runtime *in;
1219         struct port_out_runtime *out;
1220         struct instruction **action_instructions;
1221         struct rte_swx_table_state *table_state;
1222         struct table_statistics *table_stats;
1223         struct regarray_runtime *regarray_runtime;
1224         struct metarray_runtime *metarray_runtime;
1225         struct instruction *instructions;
1226         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1227
1228         uint32_t n_structs;
1229         uint32_t n_ports_in;
1230         uint32_t n_ports_out;
1231         uint32_t n_extern_objs;
1232         uint32_t n_extern_funcs;
1233         uint32_t n_actions;
1234         uint32_t n_tables;
1235         uint32_t n_regarrays;
1236         uint32_t n_metarrays;
1237         uint32_t n_headers;
1238         uint32_t thread_id;
1239         uint32_t port_id;
1240         uint32_t n_instructions;
1241         int build_done;
1242         int numa_node;
1243 };
1244
1245 /*
1246  * Struct.
1247  */
1248 static struct struct_type *
1249 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1250 {
1251         struct struct_type *elem;
1252
1253         TAILQ_FOREACH(elem, &p->struct_types, node)
1254                 if (strcmp(elem->name, name) == 0)
1255                         return elem;
1256
1257         return NULL;
1258 }
1259
1260 static struct field *
1261 struct_type_field_find(struct struct_type *st, const char *name)
1262 {
1263         uint32_t i;
1264
1265         for (i = 0; i < st->n_fields; i++) {
1266                 struct field *f = &st->fields[i];
1267
1268                 if (strcmp(f->name, name) == 0)
1269                         return f;
1270         }
1271
1272         return NULL;
1273 }
1274
1275 int
1276 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1277                                       const char *name,
1278                                       struct rte_swx_field_params *fields,
1279                                       uint32_t n_fields)
1280 {
1281         struct struct_type *st;
1282         uint32_t i;
1283
1284         CHECK(p, EINVAL);
1285         CHECK_NAME(name, EINVAL);
1286         CHECK(fields, EINVAL);
1287         CHECK(n_fields, EINVAL);
1288
1289         for (i = 0; i < n_fields; i++) {
1290                 struct rte_swx_field_params *f = &fields[i];
1291                 uint32_t j;
1292
1293                 CHECK_NAME(f->name, EINVAL);
1294                 CHECK(f->n_bits, EINVAL);
1295                 CHECK(f->n_bits <= 64, EINVAL);
1296                 CHECK((f->n_bits & 7) == 0, EINVAL);
1297
1298                 for (j = 0; j < i; j++) {
1299                         struct rte_swx_field_params *f_prev = &fields[j];
1300
1301                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
1302                 }
1303         }
1304
1305         CHECK(!struct_type_find(p, name), EEXIST);
1306
1307         /* Node allocation. */
1308         st = calloc(1, sizeof(struct struct_type));
1309         CHECK(st, ENOMEM);
1310
1311         st->fields = calloc(n_fields, sizeof(struct field));
1312         if (!st->fields) {
1313                 free(st);
1314                 CHECK(0, ENOMEM);
1315         }
1316
1317         /* Node initialization. */
1318         strcpy(st->name, name);
1319         for (i = 0; i < n_fields; i++) {
1320                 struct field *dst = &st->fields[i];
1321                 struct rte_swx_field_params *src = &fields[i];
1322
1323                 strcpy(dst->name, src->name);
1324                 dst->n_bits = src->n_bits;
1325                 dst->offset = st->n_bits;
1326
1327                 st->n_bits += src->n_bits;
1328         }
1329         st->n_fields = n_fields;
1330
1331         /* Node add to tailq. */
1332         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1333
1334         return 0;
1335 }
1336
1337 static int
1338 struct_build(struct rte_swx_pipeline *p)
1339 {
1340         uint32_t i;
1341
1342         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1343                 struct thread *t = &p->threads[i];
1344
1345                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1346                 CHECK(t->structs, ENOMEM);
1347         }
1348
1349         return 0;
1350 }
1351
1352 static void
1353 struct_build_free(struct rte_swx_pipeline *p)
1354 {
1355         uint32_t i;
1356
1357         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1358                 struct thread *t = &p->threads[i];
1359
1360                 free(t->structs);
1361                 t->structs = NULL;
1362         }
1363 }
1364
1365 static void
1366 struct_free(struct rte_swx_pipeline *p)
1367 {
1368         struct_build_free(p);
1369
1370         /* Struct types. */
1371         for ( ; ; ) {
1372                 struct struct_type *elem;
1373
1374                 elem = TAILQ_FIRST(&p->struct_types);
1375                 if (!elem)
1376                         break;
1377
1378                 TAILQ_REMOVE(&p->struct_types, elem, node);
1379                 free(elem->fields);
1380                 free(elem);
1381         }
1382 }
1383
1384 /*
1385  * Input port.
1386  */
1387 static struct port_in_type *
1388 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1389 {
1390         struct port_in_type *elem;
1391
1392         if (!name)
1393                 return NULL;
1394
1395         TAILQ_FOREACH(elem, &p->port_in_types, node)
1396                 if (strcmp(elem->name, name) == 0)
1397                         return elem;
1398
1399         return NULL;
1400 }
1401
1402 int
1403 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1404                                        const char *name,
1405                                        struct rte_swx_port_in_ops *ops)
1406 {
1407         struct port_in_type *elem;
1408
1409         CHECK(p, EINVAL);
1410         CHECK_NAME(name, EINVAL);
1411         CHECK(ops, EINVAL);
1412         CHECK(ops->create, EINVAL);
1413         CHECK(ops->free, EINVAL);
1414         CHECK(ops->pkt_rx, EINVAL);
1415         CHECK(ops->stats_read, EINVAL);
1416
1417         CHECK(!port_in_type_find(p, name), EEXIST);
1418
1419         /* Node allocation. */
1420         elem = calloc(1, sizeof(struct port_in_type));
1421         CHECK(elem, ENOMEM);
1422
1423         /* Node initialization. */
1424         strcpy(elem->name, name);
1425         memcpy(&elem->ops, ops, sizeof(*ops));
1426
1427         /* Node add to tailq. */
1428         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1429
1430         return 0;
1431 }
1432
1433 static struct port_in *
1434 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1435 {
1436         struct port_in *port;
1437
1438         TAILQ_FOREACH(port, &p->ports_in, node)
1439                 if (port->id == port_id)
1440                         return port;
1441
1442         return NULL;
1443 }
1444
1445 int
1446 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1447                                 uint32_t port_id,
1448                                 const char *port_type_name,
1449                                 void *args)
1450 {
1451         struct port_in_type *type = NULL;
1452         struct port_in *port = NULL;
1453         void *obj = NULL;
1454
1455         CHECK(p, EINVAL);
1456
1457         CHECK(!port_in_find(p, port_id), EINVAL);
1458
1459         CHECK_NAME(port_type_name, EINVAL);
1460         type = port_in_type_find(p, port_type_name);
1461         CHECK(type, EINVAL);
1462
1463         obj = type->ops.create(args);
1464         CHECK(obj, ENODEV);
1465
1466         /* Node allocation. */
1467         port = calloc(1, sizeof(struct port_in));
1468         CHECK(port, ENOMEM);
1469
1470         /* Node initialization. */
1471         port->type = type;
1472         port->obj = obj;
1473         port->id = port_id;
1474
1475         /* Node add to tailq. */
1476         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1477         if (p->n_ports_in < port_id + 1)
1478                 p->n_ports_in = port_id + 1;
1479
1480         return 0;
1481 }
1482
1483 static int
1484 port_in_build(struct rte_swx_pipeline *p)
1485 {
1486         struct port_in *port;
1487         uint32_t i;
1488
1489         CHECK(p->n_ports_in, EINVAL);
1490         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1491
1492         for (i = 0; i < p->n_ports_in; i++)
1493                 CHECK(port_in_find(p, i), EINVAL);
1494
1495         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1496         CHECK(p->in, ENOMEM);
1497
1498         TAILQ_FOREACH(port, &p->ports_in, node) {
1499                 struct port_in_runtime *in = &p->in[port->id];
1500
1501                 in->pkt_rx = port->type->ops.pkt_rx;
1502                 in->obj = port->obj;
1503         }
1504
1505         return 0;
1506 }
1507
1508 static void
1509 port_in_build_free(struct rte_swx_pipeline *p)
1510 {
1511         free(p->in);
1512         p->in = NULL;
1513 }
1514
1515 static void
1516 port_in_free(struct rte_swx_pipeline *p)
1517 {
1518         port_in_build_free(p);
1519
1520         /* Input ports. */
1521         for ( ; ; ) {
1522                 struct port_in *port;
1523
1524                 port = TAILQ_FIRST(&p->ports_in);
1525                 if (!port)
1526                         break;
1527
1528                 TAILQ_REMOVE(&p->ports_in, port, node);
1529                 port->type->ops.free(port->obj);
1530                 free(port);
1531         }
1532
1533         /* Input port types. */
1534         for ( ; ; ) {
1535                 struct port_in_type *elem;
1536
1537                 elem = TAILQ_FIRST(&p->port_in_types);
1538                 if (!elem)
1539                         break;
1540
1541                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1542                 free(elem);
1543         }
1544 }
1545
1546 /*
1547  * Output port.
1548  */
1549 static struct port_out_type *
1550 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1551 {
1552         struct port_out_type *elem;
1553
1554         if (!name)
1555                 return NULL;
1556
1557         TAILQ_FOREACH(elem, &p->port_out_types, node)
1558                 if (!strcmp(elem->name, name))
1559                         return elem;
1560
1561         return NULL;
1562 }
1563
1564 int
1565 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1566                                         const char *name,
1567                                         struct rte_swx_port_out_ops *ops)
1568 {
1569         struct port_out_type *elem;
1570
1571         CHECK(p, EINVAL);
1572         CHECK_NAME(name, EINVAL);
1573         CHECK(ops, EINVAL);
1574         CHECK(ops->create, EINVAL);
1575         CHECK(ops->free, EINVAL);
1576         CHECK(ops->pkt_tx, EINVAL);
1577         CHECK(ops->stats_read, EINVAL);
1578
1579         CHECK(!port_out_type_find(p, name), EEXIST);
1580
1581         /* Node allocation. */
1582         elem = calloc(1, sizeof(struct port_out_type));
1583         CHECK(elem, ENOMEM);
1584
1585         /* Node initialization. */
1586         strcpy(elem->name, name);
1587         memcpy(&elem->ops, ops, sizeof(*ops));
1588
1589         /* Node add to tailq. */
1590         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1591
1592         return 0;
1593 }
1594
1595 static struct port_out *
1596 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1597 {
1598         struct port_out *port;
1599
1600         TAILQ_FOREACH(port, &p->ports_out, node)
1601                 if (port->id == port_id)
1602                         return port;
1603
1604         return NULL;
1605 }
1606
1607 int
1608 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1609                                  uint32_t port_id,
1610                                  const char *port_type_name,
1611                                  void *args)
1612 {
1613         struct port_out_type *type = NULL;
1614         struct port_out *port = NULL;
1615         void *obj = NULL;
1616
1617         CHECK(p, EINVAL);
1618
1619         CHECK(!port_out_find(p, port_id), EINVAL);
1620
1621         CHECK_NAME(port_type_name, EINVAL);
1622         type = port_out_type_find(p, port_type_name);
1623         CHECK(type, EINVAL);
1624
1625         obj = type->ops.create(args);
1626         CHECK(obj, ENODEV);
1627
1628         /* Node allocation. */
1629         port = calloc(1, sizeof(struct port_out));
1630         CHECK(port, ENOMEM);
1631
1632         /* Node initialization. */
1633         port->type = type;
1634         port->obj = obj;
1635         port->id = port_id;
1636
1637         /* Node add to tailq. */
1638         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1639         if (p->n_ports_out < port_id + 1)
1640                 p->n_ports_out = port_id + 1;
1641
1642         return 0;
1643 }
1644
1645 static int
1646 port_out_build(struct rte_swx_pipeline *p)
1647 {
1648         struct port_out *port;
1649         uint32_t i;
1650
1651         CHECK(p->n_ports_out, EINVAL);
1652
1653         for (i = 0; i < p->n_ports_out; i++)
1654                 CHECK(port_out_find(p, i), EINVAL);
1655
1656         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1657         CHECK(p->out, ENOMEM);
1658
1659         TAILQ_FOREACH(port, &p->ports_out, node) {
1660                 struct port_out_runtime *out = &p->out[port->id];
1661
1662                 out->pkt_tx = port->type->ops.pkt_tx;
1663                 out->flush = port->type->ops.flush;
1664                 out->obj = port->obj;
1665         }
1666
1667         return 0;
1668 }
1669
1670 static void
1671 port_out_build_free(struct rte_swx_pipeline *p)
1672 {
1673         free(p->out);
1674         p->out = NULL;
1675 }
1676
1677 static void
1678 port_out_free(struct rte_swx_pipeline *p)
1679 {
1680         port_out_build_free(p);
1681
1682         /* Output ports. */
1683         for ( ; ; ) {
1684                 struct port_out *port;
1685
1686                 port = TAILQ_FIRST(&p->ports_out);
1687                 if (!port)
1688                         break;
1689
1690                 TAILQ_REMOVE(&p->ports_out, port, node);
1691                 port->type->ops.free(port->obj);
1692                 free(port);
1693         }
1694
1695         /* Output port types. */
1696         for ( ; ; ) {
1697                 struct port_out_type *elem;
1698
1699                 elem = TAILQ_FIRST(&p->port_out_types);
1700                 if (!elem)
1701                         break;
1702
1703                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1704                 free(elem);
1705         }
1706 }
1707
1708 /*
1709  * Extern object.
1710  */
1711 static struct extern_type *
1712 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1713 {
1714         struct extern_type *elem;
1715
1716         TAILQ_FOREACH(elem, &p->extern_types, node)
1717                 if (strcmp(elem->name, name) == 0)
1718                         return elem;
1719
1720         return NULL;
1721 }
1722
1723 static struct extern_type_member_func *
1724 extern_type_member_func_find(struct extern_type *type, const char *name)
1725 {
1726         struct extern_type_member_func *elem;
1727
1728         TAILQ_FOREACH(elem, &type->funcs, node)
1729                 if (strcmp(elem->name, name) == 0)
1730                         return elem;
1731
1732         return NULL;
1733 }
1734
1735 static struct extern_obj *
1736 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1737 {
1738         struct extern_obj *elem;
1739
1740         TAILQ_FOREACH(elem, &p->extern_objs, node)
1741                 if (strcmp(elem->name, name) == 0)
1742                         return elem;
1743
1744         return NULL;
1745 }
1746
1747 static struct extern_type_member_func *
1748 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1749                              const char *name,
1750                              struct extern_obj **obj)
1751 {
1752         struct extern_obj *object;
1753         struct extern_type_member_func *func;
1754         char *object_name, *func_name;
1755
1756         if (name[0] != 'e' || name[1] != '.')
1757                 return NULL;
1758
1759         object_name = strdup(&name[2]);
1760         if (!object_name)
1761                 return NULL;
1762
1763         func_name = strchr(object_name, '.');
1764         if (!func_name) {
1765                 free(object_name);
1766                 return NULL;
1767         }
1768
1769         *func_name = 0;
1770         func_name++;
1771
1772         object = extern_obj_find(p, object_name);
1773         if (!object) {
1774                 free(object_name);
1775                 return NULL;
1776         }
1777
1778         func = extern_type_member_func_find(object->type, func_name);
1779         if (!func) {
1780                 free(object_name);
1781                 return NULL;
1782         }
1783
1784         if (obj)
1785                 *obj = object;
1786
1787         free(object_name);
1788         return func;
1789 }
1790
1791 static struct field *
1792 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1793                                const char *name,
1794                                struct extern_obj **object)
1795 {
1796         struct extern_obj *obj;
1797         struct field *f;
1798         char *obj_name, *field_name;
1799
1800         if ((name[0] != 'e') || (name[1] != '.'))
1801                 return NULL;
1802
1803         obj_name = strdup(&name[2]);
1804         if (!obj_name)
1805                 return NULL;
1806
1807         field_name = strchr(obj_name, '.');
1808         if (!field_name) {
1809                 free(obj_name);
1810                 return NULL;
1811         }
1812
1813         *field_name = 0;
1814         field_name++;
1815
1816         obj = extern_obj_find(p, obj_name);
1817         if (!obj) {
1818                 free(obj_name);
1819                 return NULL;
1820         }
1821
1822         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1823         if (!f) {
1824                 free(obj_name);
1825                 return NULL;
1826         }
1827
1828         if (object)
1829                 *object = obj;
1830
1831         free(obj_name);
1832         return f;
1833 }
1834
1835 int
1836 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1837         const char *name,
1838         const char *mailbox_struct_type_name,
1839         rte_swx_extern_type_constructor_t constructor,
1840         rte_swx_extern_type_destructor_t destructor)
1841 {
1842         struct extern_type *elem;
1843         struct struct_type *mailbox_struct_type;
1844
1845         CHECK(p, EINVAL);
1846
1847         CHECK_NAME(name, EINVAL);
1848         CHECK(!extern_type_find(p, name), EEXIST);
1849
1850         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1851         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1852         CHECK(mailbox_struct_type, EINVAL);
1853
1854         CHECK(constructor, EINVAL);
1855         CHECK(destructor, EINVAL);
1856
1857         /* Node allocation. */
1858         elem = calloc(1, sizeof(struct extern_type));
1859         CHECK(elem, ENOMEM);
1860
1861         /* Node initialization. */
1862         strcpy(elem->name, name);
1863         elem->mailbox_struct_type = mailbox_struct_type;
1864         elem->constructor = constructor;
1865         elem->destructor = destructor;
1866         TAILQ_INIT(&elem->funcs);
1867
1868         /* Node add to tailq. */
1869         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1870
1871         return 0;
1872 }
1873
1874 int
1875 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1876         const char *extern_type_name,
1877         const char *name,
1878         rte_swx_extern_type_member_func_t member_func)
1879 {
1880         struct extern_type *type;
1881         struct extern_type_member_func *type_member;
1882
1883         CHECK(p, EINVAL);
1884
1885         CHECK_NAME(extern_type_name, EINVAL);
1886         type = extern_type_find(p, extern_type_name);
1887         CHECK(type, EINVAL);
1888         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1889
1890         CHECK_NAME(name, EINVAL);
1891         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1892
1893         CHECK(member_func, EINVAL);
1894
1895         /* Node allocation. */
1896         type_member = calloc(1, sizeof(struct extern_type_member_func));
1897         CHECK(type_member, ENOMEM);
1898
1899         /* Node initialization. */
1900         strcpy(type_member->name, name);
1901         type_member->func = member_func;
1902         type_member->id = type->n_funcs;
1903
1904         /* Node add to tailq. */
1905         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1906         type->n_funcs++;
1907
1908         return 0;
1909 }
1910
1911 int
1912 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1913                                       const char *extern_type_name,
1914                                       const char *name,
1915                                       const char *args)
1916 {
1917         struct extern_type *type;
1918         struct extern_obj *obj;
1919         void *obj_handle;
1920
1921         CHECK(p, EINVAL);
1922
1923         CHECK_NAME(extern_type_name, EINVAL);
1924         type = extern_type_find(p, extern_type_name);
1925         CHECK(type, EINVAL);
1926
1927         CHECK_NAME(name, EINVAL);
1928         CHECK(!extern_obj_find(p, name), EEXIST);
1929
1930         /* Node allocation. */
1931         obj = calloc(1, sizeof(struct extern_obj));
1932         CHECK(obj, ENOMEM);
1933
1934         /* Object construction. */
1935         obj_handle = type->constructor(args);
1936         if (!obj_handle) {
1937                 free(obj);
1938                 CHECK(0, ENODEV);
1939         }
1940
1941         /* Node initialization. */
1942         strcpy(obj->name, name);
1943         obj->type = type;
1944         obj->obj = obj_handle;
1945         obj->struct_id = p->n_structs;
1946         obj->id = p->n_extern_objs;
1947
1948         /* Node add to tailq. */
1949         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1950         p->n_extern_objs++;
1951         p->n_structs++;
1952
1953         return 0;
1954 }
1955
1956 static int
1957 extern_obj_build(struct rte_swx_pipeline *p)
1958 {
1959         uint32_t i;
1960
1961         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1962                 struct thread *t = &p->threads[i];
1963                 struct extern_obj *obj;
1964
1965                 t->extern_objs = calloc(p->n_extern_objs,
1966                                         sizeof(struct extern_obj_runtime));
1967                 CHECK(t->extern_objs, ENOMEM);
1968
1969                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1970                         struct extern_obj_runtime *r =
1971                                 &t->extern_objs[obj->id];
1972                         struct extern_type_member_func *func;
1973                         uint32_t mailbox_size =
1974                                 obj->type->mailbox_struct_type->n_bits / 8;
1975
1976                         r->obj = obj->obj;
1977
1978                         r->mailbox = calloc(1, mailbox_size);
1979                         CHECK(r->mailbox, ENOMEM);
1980
1981                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1982                                 r->funcs[func->id] = func->func;
1983
1984                         t->structs[obj->struct_id] = r->mailbox;
1985                 }
1986         }
1987
1988         return 0;
1989 }
1990
1991 static void
1992 extern_obj_build_free(struct rte_swx_pipeline *p)
1993 {
1994         uint32_t i;
1995
1996         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1997                 struct thread *t = &p->threads[i];
1998                 uint32_t j;
1999
2000                 if (!t->extern_objs)
2001                         continue;
2002
2003                 for (j = 0; j < p->n_extern_objs; j++) {
2004                         struct extern_obj_runtime *r = &t->extern_objs[j];
2005
2006                         free(r->mailbox);
2007                 }
2008
2009                 free(t->extern_objs);
2010                 t->extern_objs = NULL;
2011         }
2012 }
2013
2014 static void
2015 extern_obj_free(struct rte_swx_pipeline *p)
2016 {
2017         extern_obj_build_free(p);
2018
2019         /* Extern objects. */
2020         for ( ; ; ) {
2021                 struct extern_obj *elem;
2022
2023                 elem = TAILQ_FIRST(&p->extern_objs);
2024                 if (!elem)
2025                         break;
2026
2027                 TAILQ_REMOVE(&p->extern_objs, elem, node);
2028                 if (elem->obj)
2029                         elem->type->destructor(elem->obj);
2030                 free(elem);
2031         }
2032
2033         /* Extern types. */
2034         for ( ; ; ) {
2035                 struct extern_type *elem;
2036
2037                 elem = TAILQ_FIRST(&p->extern_types);
2038                 if (!elem)
2039                         break;
2040
2041                 TAILQ_REMOVE(&p->extern_types, elem, node);
2042
2043                 for ( ; ; ) {
2044                         struct extern_type_member_func *func;
2045
2046                         func = TAILQ_FIRST(&elem->funcs);
2047                         if (!func)
2048                                 break;
2049
2050                         TAILQ_REMOVE(&elem->funcs, func, node);
2051                         free(func);
2052                 }
2053
2054                 free(elem);
2055         }
2056 }
2057
2058 /*
2059  * Extern function.
2060  */
2061 static struct extern_func *
2062 extern_func_find(struct rte_swx_pipeline *p, const char *name)
2063 {
2064         struct extern_func *elem;
2065
2066         TAILQ_FOREACH(elem, &p->extern_funcs, node)
2067                 if (strcmp(elem->name, name) == 0)
2068                         return elem;
2069
2070         return NULL;
2071 }
2072
2073 static struct extern_func *
2074 extern_func_parse(struct rte_swx_pipeline *p,
2075                   const char *name)
2076 {
2077         if (name[0] != 'f' || name[1] != '.')
2078                 return NULL;
2079
2080         return extern_func_find(p, &name[2]);
2081 }
2082
2083 static struct field *
2084 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
2085                                 const char *name,
2086                                 struct extern_func **function)
2087 {
2088         struct extern_func *func;
2089         struct field *f;
2090         char *func_name, *field_name;
2091
2092         if ((name[0] != 'f') || (name[1] != '.'))
2093                 return NULL;
2094
2095         func_name = strdup(&name[2]);
2096         if (!func_name)
2097                 return NULL;
2098
2099         field_name = strchr(func_name, '.');
2100         if (!field_name) {
2101                 free(func_name);
2102                 return NULL;
2103         }
2104
2105         *field_name = 0;
2106         field_name++;
2107
2108         func = extern_func_find(p, func_name);
2109         if (!func) {
2110                 free(func_name);
2111                 return NULL;
2112         }
2113
2114         f = struct_type_field_find(func->mailbox_struct_type, field_name);
2115         if (!f) {
2116                 free(func_name);
2117                 return NULL;
2118         }
2119
2120         if (function)
2121                 *function = func;
2122
2123         free(func_name);
2124         return f;
2125 }
2126
2127 int
2128 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
2129                                       const char *name,
2130                                       const char *mailbox_struct_type_name,
2131                                       rte_swx_extern_func_t func)
2132 {
2133         struct extern_func *f;
2134         struct struct_type *mailbox_struct_type;
2135
2136         CHECK(p, EINVAL);
2137
2138         CHECK_NAME(name, EINVAL);
2139         CHECK(!extern_func_find(p, name), EEXIST);
2140
2141         CHECK_NAME(mailbox_struct_type_name, EINVAL);
2142         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2143         CHECK(mailbox_struct_type, EINVAL);
2144
2145         CHECK(func, EINVAL);
2146
2147         /* Node allocation. */
2148         f = calloc(1, sizeof(struct extern_func));
2149         CHECK(func, ENOMEM);
2150
2151         /* Node initialization. */
2152         strcpy(f->name, name);
2153         f->mailbox_struct_type = mailbox_struct_type;
2154         f->func = func;
2155         f->struct_id = p->n_structs;
2156         f->id = p->n_extern_funcs;
2157
2158         /* Node add to tailq. */
2159         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
2160         p->n_extern_funcs++;
2161         p->n_structs++;
2162
2163         return 0;
2164 }
2165
2166 static int
2167 extern_func_build(struct rte_swx_pipeline *p)
2168 {
2169         uint32_t i;
2170
2171         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2172                 struct thread *t = &p->threads[i];
2173                 struct extern_func *func;
2174
2175                 /* Memory allocation. */
2176                 t->extern_funcs = calloc(p->n_extern_funcs,
2177                                          sizeof(struct extern_func_runtime));
2178                 CHECK(t->extern_funcs, ENOMEM);
2179
2180                 /* Extern function. */
2181                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
2182                         struct extern_func_runtime *r =
2183                                 &t->extern_funcs[func->id];
2184                         uint32_t mailbox_size =
2185                                 func->mailbox_struct_type->n_bits / 8;
2186
2187                         r->func = func->func;
2188
2189                         r->mailbox = calloc(1, mailbox_size);
2190                         CHECK(r->mailbox, ENOMEM);
2191
2192                         t->structs[func->struct_id] = r->mailbox;
2193                 }
2194         }
2195
2196         return 0;
2197 }
2198
2199 static void
2200 extern_func_build_free(struct rte_swx_pipeline *p)
2201 {
2202         uint32_t i;
2203
2204         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2205                 struct thread *t = &p->threads[i];
2206                 uint32_t j;
2207
2208                 if (!t->extern_funcs)
2209                         continue;
2210
2211                 for (j = 0; j < p->n_extern_funcs; j++) {
2212                         struct extern_func_runtime *r = &t->extern_funcs[j];
2213
2214                         free(r->mailbox);
2215                 }
2216
2217                 free(t->extern_funcs);
2218                 t->extern_funcs = NULL;
2219         }
2220 }
2221
2222 static void
2223 extern_func_free(struct rte_swx_pipeline *p)
2224 {
2225         extern_func_build_free(p);
2226
2227         for ( ; ; ) {
2228                 struct extern_func *elem;
2229
2230                 elem = TAILQ_FIRST(&p->extern_funcs);
2231                 if (!elem)
2232                         break;
2233
2234                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
2235                 free(elem);
2236         }
2237 }
2238
2239 /*
2240  * Header.
2241  */
2242 static struct header *
2243 header_find(struct rte_swx_pipeline *p, const char *name)
2244 {
2245         struct header *elem;
2246
2247         TAILQ_FOREACH(elem, &p->headers, node)
2248                 if (strcmp(elem->name, name) == 0)
2249                         return elem;
2250
2251         return NULL;
2252 }
2253
2254 static struct header *
2255 header_parse(struct rte_swx_pipeline *p,
2256              const char *name)
2257 {
2258         if (name[0] != 'h' || name[1] != '.')
2259                 return NULL;
2260
2261         return header_find(p, &name[2]);
2262 }
2263
2264 static struct field *
2265 header_field_parse(struct rte_swx_pipeline *p,
2266                    const char *name,
2267                    struct header **header)
2268 {
2269         struct header *h;
2270         struct field *f;
2271         char *header_name, *field_name;
2272
2273         if ((name[0] != 'h') || (name[1] != '.'))
2274                 return NULL;
2275
2276         header_name = strdup(&name[2]);
2277         if (!header_name)
2278                 return NULL;
2279
2280         field_name = strchr(header_name, '.');
2281         if (!field_name) {
2282                 free(header_name);
2283                 return NULL;
2284         }
2285
2286         *field_name = 0;
2287         field_name++;
2288
2289         h = header_find(p, header_name);
2290         if (!h) {
2291                 free(header_name);
2292                 return NULL;
2293         }
2294
2295         f = struct_type_field_find(h->st, field_name);
2296         if (!f) {
2297                 free(header_name);
2298                 return NULL;
2299         }
2300
2301         if (header)
2302                 *header = h;
2303
2304         free(header_name);
2305         return f;
2306 }
2307
2308 int
2309 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2310                                         const char *name,
2311                                         const char *struct_type_name)
2312 {
2313         struct struct_type *st;
2314         struct header *h;
2315         size_t n_headers_max;
2316
2317         CHECK(p, EINVAL);
2318         CHECK_NAME(name, EINVAL);
2319         CHECK_NAME(struct_type_name, EINVAL);
2320
2321         CHECK(!header_find(p, name), EEXIST);
2322
2323         st = struct_type_find(p, struct_type_name);
2324         CHECK(st, EINVAL);
2325
2326         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2327         CHECK(p->n_headers < n_headers_max, ENOSPC);
2328
2329         /* Node allocation. */
2330         h = calloc(1, sizeof(struct header));
2331         CHECK(h, ENOMEM);
2332
2333         /* Node initialization. */
2334         strcpy(h->name, name);
2335         h->st = st;
2336         h->struct_id = p->n_structs;
2337         h->id = p->n_headers;
2338
2339         /* Node add to tailq. */
2340         TAILQ_INSERT_TAIL(&p->headers, h, node);
2341         p->n_headers++;
2342         p->n_structs++;
2343
2344         return 0;
2345 }
2346
2347 static int
2348 header_build(struct rte_swx_pipeline *p)
2349 {
2350         struct header *h;
2351         uint32_t n_bytes = 0, i;
2352
2353         TAILQ_FOREACH(h, &p->headers, node) {
2354                 n_bytes += h->st->n_bits / 8;
2355         }
2356
2357         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2358                 struct thread *t = &p->threads[i];
2359                 uint32_t offset = 0;
2360
2361                 t->headers = calloc(p->n_headers,
2362                                     sizeof(struct header_runtime));
2363                 CHECK(t->headers, ENOMEM);
2364
2365                 t->headers_out = calloc(p->n_headers,
2366                                         sizeof(struct header_out_runtime));
2367                 CHECK(t->headers_out, ENOMEM);
2368
2369                 t->header_storage = calloc(1, n_bytes);
2370                 CHECK(t->header_storage, ENOMEM);
2371
2372                 t->header_out_storage = calloc(1, n_bytes);
2373                 CHECK(t->header_out_storage, ENOMEM);
2374
2375                 TAILQ_FOREACH(h, &p->headers, node) {
2376                         uint8_t *header_storage;
2377
2378                         header_storage = &t->header_storage[offset];
2379                         offset += h->st->n_bits / 8;
2380
2381                         t->headers[h->id].ptr0 = header_storage;
2382                         t->structs[h->struct_id] = header_storage;
2383                 }
2384         }
2385
2386         return 0;
2387 }
2388
2389 static void
2390 header_build_free(struct rte_swx_pipeline *p)
2391 {
2392         uint32_t i;
2393
2394         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2395                 struct thread *t = &p->threads[i];
2396
2397                 free(t->headers_out);
2398                 t->headers_out = NULL;
2399
2400                 free(t->headers);
2401                 t->headers = NULL;
2402
2403                 free(t->header_out_storage);
2404                 t->header_out_storage = NULL;
2405
2406                 free(t->header_storage);
2407                 t->header_storage = NULL;
2408         }
2409 }
2410
2411 static void
2412 header_free(struct rte_swx_pipeline *p)
2413 {
2414         header_build_free(p);
2415
2416         for ( ; ; ) {
2417                 struct header *elem;
2418
2419                 elem = TAILQ_FIRST(&p->headers);
2420                 if (!elem)
2421                         break;
2422
2423                 TAILQ_REMOVE(&p->headers, elem, node);
2424                 free(elem);
2425         }
2426 }
2427
2428 /*
2429  * Meta-data.
2430  */
2431 static struct field *
2432 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2433 {
2434         if (!p->metadata_st)
2435                 return NULL;
2436
2437         if (name[0] != 'm' || name[1] != '.')
2438                 return NULL;
2439
2440         return struct_type_field_find(p->metadata_st, &name[2]);
2441 }
2442
2443 int
2444 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2445                                           const char *struct_type_name)
2446 {
2447         struct struct_type *st = NULL;
2448
2449         CHECK(p, EINVAL);
2450
2451         CHECK_NAME(struct_type_name, EINVAL);
2452         st  = struct_type_find(p, struct_type_name);
2453         CHECK(st, EINVAL);
2454         CHECK(!p->metadata_st, EINVAL);
2455
2456         p->metadata_st = st;
2457         p->metadata_struct_id = p->n_structs;
2458
2459         p->n_structs++;
2460
2461         return 0;
2462 }
2463
2464 static int
2465 metadata_build(struct rte_swx_pipeline *p)
2466 {
2467         uint32_t n_bytes = p->metadata_st->n_bits / 8;
2468         uint32_t i;
2469
2470         /* Thread-level initialization. */
2471         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2472                 struct thread *t = &p->threads[i];
2473                 uint8_t *metadata;
2474
2475                 metadata = calloc(1, n_bytes);
2476                 CHECK(metadata, ENOMEM);
2477
2478                 t->metadata = metadata;
2479                 t->structs[p->metadata_struct_id] = metadata;
2480         }
2481
2482         return 0;
2483 }
2484
2485 static void
2486 metadata_build_free(struct rte_swx_pipeline *p)
2487 {
2488         uint32_t i;
2489
2490         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2491                 struct thread *t = &p->threads[i];
2492
2493                 free(t->metadata);
2494                 t->metadata = NULL;
2495         }
2496 }
2497
2498 static void
2499 metadata_free(struct rte_swx_pipeline *p)
2500 {
2501         metadata_build_free(p);
2502 }
2503
2504 /*
2505  * Instruction.
2506  */
2507 static int
2508 instruction_is_tx(enum instruction_type type)
2509 {
2510         switch (type) {
2511         case INSTR_TX:
2512         case INSTR_TX_I:
2513                 return 1;
2514
2515         default:
2516                 return 0;
2517         }
2518 }
2519
2520 static int
2521 instruction_is_jmp(struct instruction *instr)
2522 {
2523         switch (instr->type) {
2524         case INSTR_JMP:
2525         case INSTR_JMP_VALID:
2526         case INSTR_JMP_INVALID:
2527         case INSTR_JMP_HIT:
2528         case INSTR_JMP_MISS:
2529         case INSTR_JMP_ACTION_HIT:
2530         case INSTR_JMP_ACTION_MISS:
2531         case INSTR_JMP_EQ:
2532         case INSTR_JMP_EQ_S:
2533         case INSTR_JMP_EQ_I:
2534         case INSTR_JMP_NEQ:
2535         case INSTR_JMP_NEQ_S:
2536         case INSTR_JMP_NEQ_I:
2537         case INSTR_JMP_LT:
2538         case INSTR_JMP_LT_MH:
2539         case INSTR_JMP_LT_HM:
2540         case INSTR_JMP_LT_HH:
2541         case INSTR_JMP_LT_MI:
2542         case INSTR_JMP_LT_HI:
2543         case INSTR_JMP_GT:
2544         case INSTR_JMP_GT_MH:
2545         case INSTR_JMP_GT_HM:
2546         case INSTR_JMP_GT_HH:
2547         case INSTR_JMP_GT_MI:
2548         case INSTR_JMP_GT_HI:
2549                 return 1;
2550
2551         default:
2552                 return 0;
2553         }
2554 }
2555
2556 static struct field *
2557 action_field_parse(struct action *action, const char *name);
2558
2559 static struct field *
2560 struct_field_parse(struct rte_swx_pipeline *p,
2561                    struct action *action,
2562                    const char *name,
2563                    uint32_t *struct_id)
2564 {
2565         struct field *f;
2566
2567         switch (name[0]) {
2568         case 'h':
2569         {
2570                 struct header *header;
2571
2572                 f = header_field_parse(p, name, &header);
2573                 if (!f)
2574                         return NULL;
2575
2576                 *struct_id = header->struct_id;
2577                 return f;
2578         }
2579
2580         case 'm':
2581         {
2582                 f = metadata_field_parse(p, name);
2583                 if (!f)
2584                         return NULL;
2585
2586                 *struct_id = p->metadata_struct_id;
2587                 return f;
2588         }
2589
2590         case 't':
2591         {
2592                 if (!action)
2593                         return NULL;
2594
2595                 f = action_field_parse(action, name);
2596                 if (!f)
2597                         return NULL;
2598
2599                 *struct_id = 0;
2600                 return f;
2601         }
2602
2603         case 'e':
2604         {
2605                 struct extern_obj *obj;
2606
2607                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2608                 if (!f)
2609                         return NULL;
2610
2611                 *struct_id = obj->struct_id;
2612                 return f;
2613         }
2614
2615         case 'f':
2616         {
2617                 struct extern_func *func;
2618
2619                 f = extern_func_mailbox_field_parse(p, name, &func);
2620                 if (!f)
2621                         return NULL;
2622
2623                 *struct_id = func->struct_id;
2624                 return f;
2625         }
2626
2627         default:
2628                 return NULL;
2629         }
2630 }
2631
2632 static inline void
2633 pipeline_port_inc(struct rte_swx_pipeline *p)
2634 {
2635         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2636 }
2637
2638 static inline void
2639 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2640 {
2641         t->ip = p->instructions;
2642 }
2643
2644 static inline void
2645 thread_ip_set(struct thread *t, struct instruction *ip)
2646 {
2647         t->ip = ip;
2648 }
2649
2650 static inline void
2651 thread_ip_action_call(struct rte_swx_pipeline *p,
2652                       struct thread *t,
2653                       uint32_t action_id)
2654 {
2655         t->ret = t->ip + 1;
2656         t->ip = p->action_instructions[action_id];
2657 }
2658
2659 static inline void
2660 thread_ip_inc(struct rte_swx_pipeline *p);
2661
2662 static inline void
2663 thread_ip_inc(struct rte_swx_pipeline *p)
2664 {
2665         struct thread *t = &p->threads[p->thread_id];
2666
2667         t->ip++;
2668 }
2669
2670 static inline void
2671 thread_ip_inc_cond(struct thread *t, int cond)
2672 {
2673         t->ip += cond;
2674 }
2675
2676 static inline void
2677 thread_yield(struct rte_swx_pipeline *p)
2678 {
2679         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2680 }
2681
2682 static inline void
2683 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2684 {
2685         p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2686 }
2687
2688 /*
2689  * rx.
2690  */
2691 static int
2692 instr_rx_translate(struct rte_swx_pipeline *p,
2693                    struct action *action,
2694                    char **tokens,
2695                    int n_tokens,
2696                    struct instruction *instr,
2697                    struct instruction_data *data __rte_unused)
2698 {
2699         struct field *f;
2700
2701         CHECK(!action, EINVAL);
2702         CHECK(n_tokens == 2, EINVAL);
2703
2704         f = metadata_field_parse(p, tokens[1]);
2705         CHECK(f, EINVAL);
2706
2707         instr->type = INSTR_RX;
2708         instr->io.io.offset = f->offset / 8;
2709         instr->io.io.n_bits = f->n_bits;
2710         return 0;
2711 }
2712
2713 static inline void
2714 instr_rx_exec(struct rte_swx_pipeline *p);
2715
2716 static inline void
2717 instr_rx_exec(struct rte_swx_pipeline *p)
2718 {
2719         struct thread *t = &p->threads[p->thread_id];
2720         struct instruction *ip = t->ip;
2721         struct port_in_runtime *port = &p->in[p->port_id];
2722         struct rte_swx_pkt *pkt = &t->pkt;
2723         int pkt_received;
2724
2725         /* Packet. */
2726         pkt_received = port->pkt_rx(port->obj, pkt);
2727         t->ptr = &pkt->pkt[pkt->offset];
2728         rte_prefetch0(t->ptr);
2729
2730         TRACE("[Thread %2u] rx %s from port %u\n",
2731               p->thread_id,
2732               pkt_received ? "1 pkt" : "0 pkts",
2733               p->port_id);
2734
2735         /* Headers. */
2736         t->valid_headers = 0;
2737         t->n_headers_out = 0;
2738
2739         /* Meta-data. */
2740         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2741
2742         /* Tables. */
2743         t->table_state = p->table_state;
2744
2745         /* Thread. */
2746         pipeline_port_inc(p);
2747         thread_ip_inc_cond(t, pkt_received);
2748         thread_yield(p);
2749 }
2750
2751 /*
2752  * tx.
2753  */
2754 static int
2755 instr_tx_translate(struct rte_swx_pipeline *p,
2756                    struct action *action __rte_unused,
2757                    char **tokens,
2758                    int n_tokens,
2759                    struct instruction *instr,
2760                    struct instruction_data *data __rte_unused)
2761 {
2762         char *port = tokens[1];
2763         struct field *f;
2764         uint32_t port_val;
2765
2766         CHECK(n_tokens == 2, EINVAL);
2767
2768         f = metadata_field_parse(p, port);
2769         if (f) {
2770                 instr->type = INSTR_TX;
2771                 instr->io.io.offset = f->offset / 8;
2772                 instr->io.io.n_bits = f->n_bits;
2773                 return 0;
2774         }
2775
2776         /* TX_I. */
2777         port_val = strtoul(port, &port, 0);
2778         CHECK(!port[0], EINVAL);
2779
2780         instr->type = INSTR_TX_I;
2781         instr->io.io.val = port_val;
2782         return 0;
2783 }
2784
2785 static int
2786 instr_drop_translate(struct rte_swx_pipeline *p,
2787                      struct action *action __rte_unused,
2788                      char **tokens __rte_unused,
2789                      int n_tokens,
2790                      struct instruction *instr,
2791                      struct instruction_data *data __rte_unused)
2792 {
2793         CHECK(n_tokens == 1, EINVAL);
2794
2795         /* TX_I. */
2796         instr->type = INSTR_TX_I;
2797         instr->io.io.val = p->n_ports_out - 1;
2798         return 0;
2799 }
2800
2801 static inline void
2802 emit_handler(struct thread *t)
2803 {
2804         struct header_out_runtime *h0 = &t->headers_out[0];
2805         struct header_out_runtime *h1 = &t->headers_out[1];
2806         uint32_t offset = 0, i;
2807
2808         /* No header change or header decapsulation. */
2809         if ((t->n_headers_out == 1) &&
2810             (h0->ptr + h0->n_bytes == t->ptr)) {
2811                 TRACE("Emit handler: no header change or header decap.\n");
2812
2813                 t->pkt.offset -= h0->n_bytes;
2814                 t->pkt.length += h0->n_bytes;
2815
2816                 return;
2817         }
2818
2819         /* Header encapsulation (optionally, with prior header decasulation). */
2820         if ((t->n_headers_out == 2) &&
2821             (h1->ptr + h1->n_bytes == t->ptr) &&
2822             (h0->ptr == h0->ptr0)) {
2823                 uint32_t offset;
2824
2825                 TRACE("Emit handler: header encapsulation.\n");
2826
2827                 offset = h0->n_bytes + h1->n_bytes;
2828                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2829                 t->pkt.offset -= offset;
2830                 t->pkt.length += offset;
2831
2832                 return;
2833         }
2834
2835         /* Header insertion. */
2836         /* TBD */
2837
2838         /* Header extraction. */
2839         /* TBD */
2840
2841         /* For any other case. */
2842         TRACE("Emit handler: complex case.\n");
2843
2844         for (i = 0; i < t->n_headers_out; i++) {
2845                 struct header_out_runtime *h = &t->headers_out[i];
2846
2847                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2848                 offset += h->n_bytes;
2849         }
2850
2851         if (offset) {
2852                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2853                 t->pkt.offset -= offset;
2854                 t->pkt.length += offset;
2855         }
2856 }
2857
2858 static inline void
2859 instr_tx_exec(struct rte_swx_pipeline *p);
2860
2861 static inline void
2862 instr_tx_exec(struct rte_swx_pipeline *p)
2863 {
2864         struct thread *t = &p->threads[p->thread_id];
2865         struct instruction *ip = t->ip;
2866         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2867         struct port_out_runtime *port = &p->out[port_id];
2868         struct rte_swx_pkt *pkt = &t->pkt;
2869
2870         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2871               p->thread_id,
2872               (uint32_t)port_id);
2873
2874         /* Headers. */
2875         emit_handler(t);
2876
2877         /* Packet. */
2878         port->pkt_tx(port->obj, pkt);
2879
2880         /* Thread. */
2881         thread_ip_reset(p, t);
2882         instr_rx_exec(p);
2883 }
2884
2885 static inline void
2886 instr_tx_i_exec(struct rte_swx_pipeline *p)
2887 {
2888         struct thread *t = &p->threads[p->thread_id];
2889         struct instruction *ip = t->ip;
2890         uint64_t port_id = ip->io.io.val;
2891         struct port_out_runtime *port = &p->out[port_id];
2892         struct rte_swx_pkt *pkt = &t->pkt;
2893
2894         TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
2895               p->thread_id,
2896               (uint32_t)port_id);
2897
2898         /* Headers. */
2899         emit_handler(t);
2900
2901         /* Packet. */
2902         port->pkt_tx(port->obj, pkt);
2903
2904         /* Thread. */
2905         thread_ip_reset(p, t);
2906         instr_rx_exec(p);
2907 }
2908
2909 /*
2910  * extract.
2911  */
2912 static int
2913 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2914                             struct action *action,
2915                             char **tokens,
2916                             int n_tokens,
2917                             struct instruction *instr,
2918                             struct instruction_data *data __rte_unused)
2919 {
2920         struct header *h;
2921
2922         CHECK(!action, EINVAL);
2923         CHECK(n_tokens == 2, EINVAL);
2924
2925         h = header_parse(p, tokens[1]);
2926         CHECK(h, EINVAL);
2927
2928         instr->type = INSTR_HDR_EXTRACT;
2929         instr->io.hdr.header_id[0] = h->id;
2930         instr->io.hdr.struct_id[0] = h->struct_id;
2931         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2932         return 0;
2933 }
2934
2935 static inline void
2936 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2937
2938 static inline void
2939 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2940 {
2941         struct thread *t = &p->threads[p->thread_id];
2942         struct instruction *ip = t->ip;
2943         uint64_t valid_headers = t->valid_headers;
2944         uint8_t *ptr = t->ptr;
2945         uint32_t offset = t->pkt.offset;
2946         uint32_t length = t->pkt.length;
2947         uint32_t i;
2948
2949         for (i = 0; i < n_extract; i++) {
2950                 uint32_t header_id = ip->io.hdr.header_id[i];
2951                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2952                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2953
2954                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2955                       p->thread_id,
2956                       header_id,
2957                       n_bytes);
2958
2959                 /* Headers. */
2960                 t->structs[struct_id] = ptr;
2961                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2962
2963                 /* Packet. */
2964                 offset += n_bytes;
2965                 length -= n_bytes;
2966                 ptr += n_bytes;
2967         }
2968
2969         /* Headers. */
2970         t->valid_headers = valid_headers;
2971
2972         /* Packet. */
2973         t->pkt.offset = offset;
2974         t->pkt.length = length;
2975         t->ptr = ptr;
2976 }
2977
2978 static inline void
2979 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2980 {
2981         __instr_hdr_extract_exec(p, 1);
2982
2983         /* Thread. */
2984         thread_ip_inc(p);
2985 }
2986
2987 static inline void
2988 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2989 {
2990         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2991               p->thread_id);
2992
2993         __instr_hdr_extract_exec(p, 2);
2994
2995         /* Thread. */
2996         thread_ip_inc(p);
2997 }
2998
2999 static inline void
3000 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
3001 {
3002         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3003               p->thread_id);
3004
3005         __instr_hdr_extract_exec(p, 3);
3006
3007         /* Thread. */
3008         thread_ip_inc(p);
3009 }
3010
3011 static inline void
3012 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
3013 {
3014         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3015               p->thread_id);
3016
3017         __instr_hdr_extract_exec(p, 4);
3018
3019         /* Thread. */
3020         thread_ip_inc(p);
3021 }
3022
3023 static inline void
3024 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
3025 {
3026         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3027               p->thread_id);
3028
3029         __instr_hdr_extract_exec(p, 5);
3030
3031         /* Thread. */
3032         thread_ip_inc(p);
3033 }
3034
3035 static inline void
3036 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
3037 {
3038         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3039               p->thread_id);
3040
3041         __instr_hdr_extract_exec(p, 6);
3042
3043         /* Thread. */
3044         thread_ip_inc(p);
3045 }
3046
3047 static inline void
3048 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
3049 {
3050         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3051               p->thread_id);
3052
3053         __instr_hdr_extract_exec(p, 7);
3054
3055         /* Thread. */
3056         thread_ip_inc(p);
3057 }
3058
3059 static inline void
3060 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
3061 {
3062         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3063               p->thread_id);
3064
3065         __instr_hdr_extract_exec(p, 8);
3066
3067         /* Thread. */
3068         thread_ip_inc(p);
3069 }
3070
3071 /*
3072  * emit.
3073  */
3074 static int
3075 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
3076                          struct action *action __rte_unused,
3077                          char **tokens,
3078                          int n_tokens,
3079                          struct instruction *instr,
3080                          struct instruction_data *data __rte_unused)
3081 {
3082         struct header *h;
3083
3084         CHECK(n_tokens == 2, EINVAL);
3085
3086         h = header_parse(p, tokens[1]);
3087         CHECK(h, EINVAL);
3088
3089         instr->type = INSTR_HDR_EMIT;
3090         instr->io.hdr.header_id[0] = h->id;
3091         instr->io.hdr.struct_id[0] = h->struct_id;
3092         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3093         return 0;
3094 }
3095
3096 static inline void
3097 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
3098
3099 static inline void
3100 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
3101 {
3102         struct thread *t = &p->threads[p->thread_id];
3103         struct instruction *ip = t->ip;
3104         uint64_t valid_headers = t->valid_headers;
3105         uint32_t n_headers_out = t->n_headers_out;
3106         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
3107         uint8_t *ho_ptr = NULL;
3108         uint32_t ho_nbytes = 0, first = 1, i;
3109
3110         for (i = 0; i < n_emit; i++) {
3111                 uint32_t header_id = ip->io.hdr.header_id[i];
3112                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3113                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3114
3115                 struct header_runtime *hi = &t->headers[header_id];
3116                 uint8_t *hi_ptr = t->structs[struct_id];
3117
3118                 if (!MASK64_BIT_GET(valid_headers, header_id))
3119                         continue;
3120
3121                 TRACE("[Thread %2u]: emit header %u\n",
3122                       p->thread_id,
3123                       header_id);
3124
3125                 /* Headers. */
3126                 if (first) {
3127                         first = 0;
3128
3129                         if (!t->n_headers_out) {
3130                                 ho = &t->headers_out[0];
3131
3132                                 ho->ptr0 = hi->ptr0;
3133                                 ho->ptr = hi_ptr;
3134
3135                                 ho_ptr = hi_ptr;
3136                                 ho_nbytes = n_bytes;
3137
3138                                 n_headers_out = 1;
3139
3140                                 continue;
3141                         } else {
3142                                 ho_ptr = ho->ptr;
3143                                 ho_nbytes = ho->n_bytes;
3144                         }
3145                 }
3146
3147                 if (ho_ptr + ho_nbytes == hi_ptr) {
3148                         ho_nbytes += n_bytes;
3149                 } else {
3150                         ho->n_bytes = ho_nbytes;
3151
3152                         ho++;
3153                         ho->ptr0 = hi->ptr0;
3154                         ho->ptr = hi_ptr;
3155
3156                         ho_ptr = hi_ptr;
3157                         ho_nbytes = n_bytes;
3158
3159                         n_headers_out++;
3160                 }
3161         }
3162
3163         ho->n_bytes = ho_nbytes;
3164         t->n_headers_out = n_headers_out;
3165 }
3166
3167 static inline void
3168 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
3169 {
3170         __instr_hdr_emit_exec(p, 1);
3171
3172         /* Thread. */
3173         thread_ip_inc(p);
3174 }
3175
3176 static inline void
3177 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
3178 {
3179         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3180               p->thread_id);
3181
3182         __instr_hdr_emit_exec(p, 1);
3183         instr_tx_exec(p);
3184 }
3185
3186 static inline void
3187 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
3188 {
3189         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3190               p->thread_id);
3191
3192         __instr_hdr_emit_exec(p, 2);
3193         instr_tx_exec(p);
3194 }
3195
3196 static inline void
3197 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
3198 {
3199         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3200               p->thread_id);
3201
3202         __instr_hdr_emit_exec(p, 3);
3203         instr_tx_exec(p);
3204 }
3205
3206 static inline void
3207 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
3208 {
3209         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3210               p->thread_id);
3211
3212         __instr_hdr_emit_exec(p, 4);
3213         instr_tx_exec(p);
3214 }
3215
3216 static inline void
3217 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
3218 {
3219         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3220               p->thread_id);
3221
3222         __instr_hdr_emit_exec(p, 5);
3223         instr_tx_exec(p);
3224 }
3225
3226 static inline void
3227 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
3228 {
3229         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3230               p->thread_id);
3231
3232         __instr_hdr_emit_exec(p, 6);
3233         instr_tx_exec(p);
3234 }
3235
3236 static inline void
3237 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
3238 {
3239         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3240               p->thread_id);
3241
3242         __instr_hdr_emit_exec(p, 7);
3243         instr_tx_exec(p);
3244 }
3245
3246 static inline void
3247 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
3248 {
3249         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
3250               p->thread_id);
3251
3252         __instr_hdr_emit_exec(p, 8);
3253         instr_tx_exec(p);
3254 }
3255
3256 /*
3257  * validate.
3258  */
3259 static int
3260 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
3261                              struct action *action __rte_unused,
3262                              char **tokens,
3263                              int n_tokens,
3264                              struct instruction *instr,
3265                              struct instruction_data *data __rte_unused)
3266 {
3267         struct header *h;
3268
3269         CHECK(n_tokens == 2, EINVAL);
3270
3271         h = header_parse(p, tokens[1]);
3272         CHECK(h, EINVAL);
3273
3274         instr->type = INSTR_HDR_VALIDATE;
3275         instr->valid.header_id = h->id;
3276         return 0;
3277 }
3278
3279 static inline void
3280 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
3281 {
3282         struct thread *t = &p->threads[p->thread_id];
3283         struct instruction *ip = t->ip;
3284         uint32_t header_id = ip->valid.header_id;
3285
3286         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
3287
3288         /* Headers. */
3289         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
3290
3291         /* Thread. */
3292         thread_ip_inc(p);
3293 }
3294
3295 /*
3296  * invalidate.
3297  */
3298 static int
3299 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
3300                                struct action *action __rte_unused,
3301                                char **tokens,
3302                                int n_tokens,
3303                                struct instruction *instr,
3304                                struct instruction_data *data __rte_unused)
3305 {
3306         struct header *h;
3307
3308         CHECK(n_tokens == 2, EINVAL);
3309
3310         h = header_parse(p, tokens[1]);
3311         CHECK(h, EINVAL);
3312
3313         instr->type = INSTR_HDR_INVALIDATE;
3314         instr->valid.header_id = h->id;
3315         return 0;
3316 }
3317
3318 static inline void
3319 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3320 {
3321         struct thread *t = &p->threads[p->thread_id];
3322         struct instruction *ip = t->ip;
3323         uint32_t header_id = ip->valid.header_id;
3324
3325         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3326
3327         /* Headers. */
3328         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3329
3330         /* Thread. */
3331         thread_ip_inc(p);
3332 }
3333
3334 /*
3335  * table.
3336  */
3337 static struct table *
3338 table_find(struct rte_swx_pipeline *p, const char *name);
3339
3340 static int
3341 instr_table_translate(struct rte_swx_pipeline *p,
3342                       struct action *action,
3343                       char **tokens,
3344                       int n_tokens,
3345                       struct instruction *instr,
3346                       struct instruction_data *data __rte_unused)
3347 {
3348         struct table *t;
3349
3350         CHECK(!action, EINVAL);
3351         CHECK(n_tokens == 2, EINVAL);
3352
3353         t = table_find(p, tokens[1]);
3354         CHECK(t, EINVAL);
3355
3356         instr->type = INSTR_TABLE;
3357         instr->table.table_id = t->id;
3358         return 0;
3359 }
3360
3361 static inline void
3362 instr_table_exec(struct rte_swx_pipeline *p)
3363 {
3364         struct thread *t = &p->threads[p->thread_id];
3365         struct instruction *ip = t->ip;
3366         uint32_t table_id = ip->table.table_id;
3367         struct rte_swx_table_state *ts = &t->table_state[table_id];
3368         struct table_runtime *table = &t->tables[table_id];
3369         struct table_statistics *stats = &p->table_stats[table_id];
3370         uint64_t action_id, n_pkts_hit, n_pkts_action;
3371         uint8_t *action_data;
3372         int done, hit;
3373
3374         /* Table. */
3375         done = table->func(ts->obj,
3376                            table->mailbox,
3377                            table->key,
3378                            &action_id,
3379                            &action_data,
3380                            &hit);
3381         if (!done) {
3382                 /* Thread. */
3383                 TRACE("[Thread %2u] table %u (not finalized)\n",
3384                       p->thread_id,
3385                       table_id);
3386
3387                 thread_yield(p);
3388                 return;
3389         }
3390
3391         action_id = hit ? action_id : ts->default_action_id;
3392         action_data = hit ? action_data : ts->default_action_data;
3393         n_pkts_hit = stats->n_pkts_hit[hit];
3394         n_pkts_action = stats->n_pkts_action[action_id];
3395
3396         TRACE("[Thread %2u] table %u (%s, action %u)\n",
3397               p->thread_id,
3398               table_id,
3399               hit ? "hit" : "miss",
3400               (uint32_t)action_id);
3401
3402         t->action_id = action_id;
3403         t->structs[0] = action_data;
3404         t->hit = hit;
3405         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
3406         stats->n_pkts_action[action_id] = n_pkts_action + 1;
3407
3408         /* Thread. */
3409         thread_ip_action_call(p, t, action_id);
3410 }
3411
3412 /*
3413  * extern.
3414  */
3415 static int
3416 instr_extern_translate(struct rte_swx_pipeline *p,
3417                        struct action *action __rte_unused,
3418                        char **tokens,
3419                        int n_tokens,
3420                        struct instruction *instr,
3421                        struct instruction_data *data __rte_unused)
3422 {
3423         char *token = tokens[1];
3424
3425         CHECK(n_tokens == 2, EINVAL);
3426
3427         if (token[0] == 'e') {
3428                 struct extern_obj *obj;
3429                 struct extern_type_member_func *func;
3430
3431                 func = extern_obj_member_func_parse(p, token, &obj);
3432                 CHECK(func, EINVAL);
3433
3434                 instr->type = INSTR_EXTERN_OBJ;
3435                 instr->ext_obj.ext_obj_id = obj->id;
3436                 instr->ext_obj.func_id = func->id;
3437
3438                 return 0;
3439         }
3440
3441         if (token[0] == 'f') {
3442                 struct extern_func *func;
3443
3444                 func = extern_func_parse(p, token);
3445                 CHECK(func, EINVAL);
3446
3447                 instr->type = INSTR_EXTERN_FUNC;
3448                 instr->ext_func.ext_func_id = func->id;
3449
3450                 return 0;
3451         }
3452
3453         CHECK(0, EINVAL);
3454 }
3455
3456 static inline void
3457 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3458 {
3459         struct thread *t = &p->threads[p->thread_id];
3460         struct instruction *ip = t->ip;
3461         uint32_t obj_id = ip->ext_obj.ext_obj_id;
3462         uint32_t func_id = ip->ext_obj.func_id;
3463         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3464         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3465
3466         TRACE("[Thread %2u] extern obj %u member func %u\n",
3467               p->thread_id,
3468               obj_id,
3469               func_id);
3470
3471         /* Extern object member function execute. */
3472         uint32_t done = func(obj->obj, obj->mailbox);
3473
3474         /* Thread. */
3475         thread_ip_inc_cond(t, done);
3476         thread_yield_cond(p, done ^ 1);
3477 }
3478
3479 static inline void
3480 instr_extern_func_exec(struct rte_swx_pipeline *p)
3481 {
3482         struct thread *t = &p->threads[p->thread_id];
3483         struct instruction *ip = t->ip;
3484         uint32_t ext_func_id = ip->ext_func.ext_func_id;
3485         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3486         rte_swx_extern_func_t func = ext_func->func;
3487
3488         TRACE("[Thread %2u] extern func %u\n",
3489               p->thread_id,
3490               ext_func_id);
3491
3492         /* Extern function execute. */
3493         uint32_t done = func(ext_func->mailbox);
3494
3495         /* Thread. */
3496         thread_ip_inc_cond(t, done);
3497         thread_yield_cond(p, done ^ 1);
3498 }
3499
3500 /*
3501  * mov.
3502  */
3503 static int
3504 instr_mov_translate(struct rte_swx_pipeline *p,
3505                     struct action *action,
3506                     char **tokens,
3507                     int n_tokens,
3508                     struct instruction *instr,
3509                     struct instruction_data *data __rte_unused)
3510 {
3511         char *dst = tokens[1], *src = tokens[2];
3512         struct field *fdst, *fsrc;
3513         uint64_t src_val;
3514         uint32_t dst_struct_id, src_struct_id;
3515
3516         CHECK(n_tokens == 3, EINVAL);
3517
3518         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3519         CHECK(fdst, EINVAL);
3520
3521         /* MOV or MOV_S. */
3522         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3523         if (fsrc) {
3524                 instr->type = INSTR_MOV;
3525                 if ((dst[0] == 'h' && src[0] != 'h') ||
3526                     (dst[0] != 'h' && src[0] == 'h'))
3527                         instr->type = INSTR_MOV_S;
3528
3529                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3530                 instr->mov.dst.n_bits = fdst->n_bits;
3531                 instr->mov.dst.offset = fdst->offset / 8;
3532                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3533                 instr->mov.src.n_bits = fsrc->n_bits;
3534                 instr->mov.src.offset = fsrc->offset / 8;
3535                 return 0;
3536         }
3537
3538         /* MOV_I. */
3539         src_val = strtoull(src, &src, 0);
3540         CHECK(!src[0], EINVAL);
3541
3542         if (dst[0] == 'h')
3543                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3544
3545         instr->type = INSTR_MOV_I;
3546         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3547         instr->mov.dst.n_bits = fdst->n_bits;
3548         instr->mov.dst.offset = fdst->offset / 8;
3549         instr->mov.src_val = src_val;
3550         return 0;
3551 }
3552
3553 static inline void
3554 instr_mov_exec(struct rte_swx_pipeline *p)
3555 {
3556         struct thread *t = &p->threads[p->thread_id];
3557         struct instruction *ip = t->ip;
3558
3559         TRACE("[Thread %2u] mov\n",
3560               p->thread_id);
3561
3562         MOV(t, ip);
3563
3564         /* Thread. */
3565         thread_ip_inc(p);
3566 }
3567
3568 static inline void
3569 instr_mov_s_exec(struct rte_swx_pipeline *p)
3570 {
3571         struct thread *t = &p->threads[p->thread_id];
3572         struct instruction *ip = t->ip;
3573
3574         TRACE("[Thread %2u] mov (s)\n",
3575               p->thread_id);
3576
3577         MOV_S(t, ip);
3578
3579         /* Thread. */
3580         thread_ip_inc(p);
3581 }
3582
3583 static inline void
3584 instr_mov_i_exec(struct rte_swx_pipeline *p)
3585 {
3586         struct thread *t = &p->threads[p->thread_id];
3587         struct instruction *ip = t->ip;
3588
3589         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
3590               p->thread_id,
3591               ip->mov.src_val);
3592
3593         MOV_I(t, ip);
3594
3595         /* Thread. */
3596         thread_ip_inc(p);
3597 }
3598
3599 /*
3600  * dma.
3601  */
3602 static int
3603 instr_dma_translate(struct rte_swx_pipeline *p,
3604                     struct action *action,
3605                     char **tokens,
3606                     int n_tokens,
3607                     struct instruction *instr,
3608                     struct instruction_data *data __rte_unused)
3609 {
3610         char *dst = tokens[1];
3611         char *src = tokens[2];
3612         struct header *h;
3613         struct field *tf;
3614
3615         CHECK(action, EINVAL);
3616         CHECK(n_tokens == 3, EINVAL);
3617
3618         h = header_parse(p, dst);
3619         CHECK(h, EINVAL);
3620
3621         tf = action_field_parse(action, src);
3622         CHECK(tf, EINVAL);
3623
3624         instr->type = INSTR_DMA_HT;
3625         instr->dma.dst.header_id[0] = h->id;
3626         instr->dma.dst.struct_id[0] = h->struct_id;
3627         instr->dma.n_bytes[0] = h->st->n_bits / 8;
3628         instr->dma.src.offset[0] = tf->offset / 8;
3629
3630         return 0;
3631 }
3632
3633 static inline void
3634 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3635
3636 static inline void
3637 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3638 {
3639         struct thread *t = &p->threads[p->thread_id];
3640         struct instruction *ip = t->ip;
3641         uint8_t *action_data = t->structs[0];
3642         uint64_t valid_headers = t->valid_headers;
3643         uint32_t i;
3644
3645         for (i = 0; i < n_dma; i++) {
3646                 uint32_t header_id = ip->dma.dst.header_id[i];
3647                 uint32_t struct_id = ip->dma.dst.struct_id[i];
3648                 uint32_t offset = ip->dma.src.offset[i];
3649                 uint32_t n_bytes = ip->dma.n_bytes[i];
3650
3651                 struct header_runtime *h = &t->headers[header_id];
3652                 uint8_t *h_ptr0 = h->ptr0;
3653                 uint8_t *h_ptr = t->structs[struct_id];
3654
3655                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3656                         h_ptr : h_ptr0;
3657                 void *src = &action_data[offset];
3658
3659                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3660
3661                 /* Headers. */
3662                 memcpy(dst, src, n_bytes);
3663                 t->structs[struct_id] = dst;
3664                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3665         }
3666
3667         t->valid_headers = valid_headers;
3668 }
3669
3670 static inline void
3671 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3672 {
3673         __instr_dma_ht_exec(p, 1);
3674
3675         /* Thread. */
3676         thread_ip_inc(p);
3677 }
3678
3679 static inline void
3680 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3681 {
3682         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3683               p->thread_id);
3684
3685         __instr_dma_ht_exec(p, 2);
3686
3687         /* Thread. */
3688         thread_ip_inc(p);
3689 }
3690
3691 static inline void
3692 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3693 {
3694         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3695               p->thread_id);
3696
3697         __instr_dma_ht_exec(p, 3);
3698
3699         /* Thread. */
3700         thread_ip_inc(p);
3701 }
3702
3703 static inline void
3704 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3705 {
3706         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3707               p->thread_id);
3708
3709         __instr_dma_ht_exec(p, 4);
3710
3711         /* Thread. */
3712         thread_ip_inc(p);
3713 }
3714
3715 static inline void
3716 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3717 {
3718         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3719               p->thread_id);
3720
3721         __instr_dma_ht_exec(p, 5);
3722
3723         /* Thread. */
3724         thread_ip_inc(p);
3725 }
3726
3727 static inline void
3728 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3729 {
3730         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3731               p->thread_id);
3732
3733         __instr_dma_ht_exec(p, 6);
3734
3735         /* Thread. */
3736         thread_ip_inc(p);
3737 }
3738
3739 static inline void
3740 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3741 {
3742         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3743               p->thread_id);
3744
3745         __instr_dma_ht_exec(p, 7);
3746
3747         /* Thread. */
3748         thread_ip_inc(p);
3749 }
3750
3751 static inline void
3752 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3753 {
3754         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3755               p->thread_id);
3756
3757         __instr_dma_ht_exec(p, 8);
3758
3759         /* Thread. */
3760         thread_ip_inc(p);
3761 }
3762
3763 /*
3764  * alu.
3765  */
3766 static int
3767 instr_alu_add_translate(struct rte_swx_pipeline *p,
3768                         struct action *action,
3769                         char **tokens,
3770                         int n_tokens,
3771                         struct instruction *instr,
3772                         struct instruction_data *data __rte_unused)
3773 {
3774         char *dst = tokens[1], *src = tokens[2];
3775         struct field *fdst, *fsrc;
3776         uint64_t src_val;
3777         uint32_t dst_struct_id, src_struct_id;
3778
3779         CHECK(n_tokens == 3, EINVAL);
3780
3781         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3782         CHECK(fdst, EINVAL);
3783
3784         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3785         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3786         if (fsrc) {
3787                 instr->type = INSTR_ALU_ADD;
3788                 if (dst[0] == 'h' && src[0] != 'h')
3789                         instr->type = INSTR_ALU_ADD_HM;
3790                 if (dst[0] != 'h' && src[0] == 'h')
3791                         instr->type = INSTR_ALU_ADD_MH;
3792                 if (dst[0] == 'h' && src[0] == 'h')
3793                         instr->type = INSTR_ALU_ADD_HH;
3794
3795                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3796                 instr->alu.dst.n_bits = fdst->n_bits;
3797                 instr->alu.dst.offset = fdst->offset / 8;
3798                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3799                 instr->alu.src.n_bits = fsrc->n_bits;
3800                 instr->alu.src.offset = fsrc->offset / 8;
3801                 return 0;
3802         }
3803
3804         /* ADD_MI, ADD_HI. */
3805         src_val = strtoull(src, &src, 0);
3806         CHECK(!src[0], EINVAL);
3807
3808         instr->type = INSTR_ALU_ADD_MI;
3809         if (dst[0] == 'h')
3810                 instr->type = INSTR_ALU_ADD_HI;
3811
3812         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3813         instr->alu.dst.n_bits = fdst->n_bits;
3814         instr->alu.dst.offset = fdst->offset / 8;
3815         instr->alu.src_val = src_val;
3816         return 0;
3817 }
3818
3819 static int
3820 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3821                         struct action *action,
3822                         char **tokens,
3823                         int n_tokens,
3824                         struct instruction *instr,
3825                         struct instruction_data *data __rte_unused)
3826 {
3827         char *dst = tokens[1], *src = tokens[2];
3828         struct field *fdst, *fsrc;
3829         uint64_t src_val;
3830         uint32_t dst_struct_id, src_struct_id;
3831
3832         CHECK(n_tokens == 3, EINVAL);
3833
3834         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3835         CHECK(fdst, EINVAL);
3836
3837         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3838         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3839         if (fsrc) {
3840                 instr->type = INSTR_ALU_SUB;
3841                 if (dst[0] == 'h' && src[0] != 'h')
3842                         instr->type = INSTR_ALU_SUB_HM;
3843                 if (dst[0] != 'h' && src[0] == 'h')
3844                         instr->type = INSTR_ALU_SUB_MH;
3845                 if (dst[0] == 'h' && src[0] == 'h')
3846                         instr->type = INSTR_ALU_SUB_HH;
3847
3848                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3849                 instr->alu.dst.n_bits = fdst->n_bits;
3850                 instr->alu.dst.offset = fdst->offset / 8;
3851                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3852                 instr->alu.src.n_bits = fsrc->n_bits;
3853                 instr->alu.src.offset = fsrc->offset / 8;
3854                 return 0;
3855         }
3856
3857         /* SUB_MI, SUB_HI. */
3858         src_val = strtoull(src, &src, 0);
3859         CHECK(!src[0], EINVAL);
3860
3861         instr->type = INSTR_ALU_SUB_MI;
3862         if (dst[0] == 'h')
3863                 instr->type = INSTR_ALU_SUB_HI;
3864
3865         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3866         instr->alu.dst.n_bits = fdst->n_bits;
3867         instr->alu.dst.offset = fdst->offset / 8;
3868         instr->alu.src_val = src_val;
3869         return 0;
3870 }
3871
3872 static int
3873 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3874                           struct action *action __rte_unused,
3875                           char **tokens,
3876                           int n_tokens,
3877                           struct instruction *instr,
3878                           struct instruction_data *data __rte_unused)
3879 {
3880         char *dst = tokens[1], *src = tokens[2];
3881         struct header *hdst, *hsrc;
3882         struct field *fdst, *fsrc;
3883
3884         CHECK(n_tokens == 3, EINVAL);
3885
3886         fdst = header_field_parse(p, dst, &hdst);
3887         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3888
3889         /* CKADD_FIELD. */
3890         fsrc = header_field_parse(p, src, &hsrc);
3891         if (fsrc) {
3892                 instr->type = INSTR_ALU_CKADD_FIELD;
3893                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3894                 instr->alu.dst.n_bits = fdst->n_bits;
3895                 instr->alu.dst.offset = fdst->offset / 8;
3896                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3897                 instr->alu.src.n_bits = fsrc->n_bits;
3898                 instr->alu.src.offset = fsrc->offset / 8;
3899                 return 0;
3900         }
3901
3902         /* CKADD_STRUCT, CKADD_STRUCT20. */
3903         hsrc = header_parse(p, src);
3904         CHECK(hsrc, EINVAL);
3905
3906         instr->type = INSTR_ALU_CKADD_STRUCT;
3907         if ((hsrc->st->n_bits / 8) == 20)
3908                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3909
3910         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3911         instr->alu.dst.n_bits = fdst->n_bits;
3912         instr->alu.dst.offset = fdst->offset / 8;
3913         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3914         instr->alu.src.n_bits = hsrc->st->n_bits;
3915         instr->alu.src.offset = 0; /* Unused. */
3916         return 0;
3917 }
3918
3919 static int
3920 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3921                           struct action *action __rte_unused,
3922                           char **tokens,
3923                           int n_tokens,
3924                           struct instruction *instr,
3925                           struct instruction_data *data __rte_unused)
3926 {
3927         char *dst = tokens[1], *src = tokens[2];
3928         struct header *hdst, *hsrc;
3929         struct field *fdst, *fsrc;
3930
3931         CHECK(n_tokens == 3, EINVAL);
3932
3933         fdst = header_field_parse(p, dst, &hdst);
3934         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3935
3936         fsrc = header_field_parse(p, src, &hsrc);
3937         CHECK(fsrc, EINVAL);
3938
3939         instr->type = INSTR_ALU_CKSUB_FIELD;
3940         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3941         instr->alu.dst.n_bits = fdst->n_bits;
3942         instr->alu.dst.offset = fdst->offset / 8;
3943         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3944         instr->alu.src.n_bits = fsrc->n_bits;
3945         instr->alu.src.offset = fsrc->offset / 8;
3946         return 0;
3947 }
3948
3949 static int
3950 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3951                         struct action *action,
3952                         char **tokens,
3953                         int n_tokens,
3954                         struct instruction *instr,
3955                         struct instruction_data *data __rte_unused)
3956 {
3957         char *dst = tokens[1], *src = tokens[2];
3958         struct field *fdst, *fsrc;
3959         uint64_t src_val;
3960         uint32_t dst_struct_id, src_struct_id;
3961
3962         CHECK(n_tokens == 3, EINVAL);
3963
3964         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3965         CHECK(fdst, EINVAL);
3966
3967         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3968         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3969         if (fsrc) {
3970                 instr->type = INSTR_ALU_SHL;
3971                 if (dst[0] == 'h' && src[0] != 'h')
3972                         instr->type = INSTR_ALU_SHL_HM;
3973                 if (dst[0] != 'h' && src[0] == 'h')
3974                         instr->type = INSTR_ALU_SHL_MH;
3975                 if (dst[0] == 'h' && src[0] == 'h')
3976                         instr->type = INSTR_ALU_SHL_HH;
3977
3978                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3979                 instr->alu.dst.n_bits = fdst->n_bits;
3980                 instr->alu.dst.offset = fdst->offset / 8;
3981                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3982                 instr->alu.src.n_bits = fsrc->n_bits;
3983                 instr->alu.src.offset = fsrc->offset / 8;
3984                 return 0;
3985         }
3986
3987         /* SHL_MI, SHL_HI. */
3988         src_val = strtoull(src, &src, 0);
3989         CHECK(!src[0], EINVAL);
3990
3991         instr->type = INSTR_ALU_SHL_MI;
3992         if (dst[0] == 'h')
3993                 instr->type = INSTR_ALU_SHL_HI;
3994
3995         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3996         instr->alu.dst.n_bits = fdst->n_bits;
3997         instr->alu.dst.offset = fdst->offset / 8;
3998         instr->alu.src_val = src_val;
3999         return 0;
4000 }
4001
4002 static int
4003 instr_alu_shr_translate(struct rte_swx_pipeline *p,
4004                         struct action *action,
4005                         char **tokens,
4006                         int n_tokens,
4007                         struct instruction *instr,
4008                         struct instruction_data *data __rte_unused)
4009 {
4010         char *dst = tokens[1], *src = tokens[2];
4011         struct field *fdst, *fsrc;
4012         uint64_t src_val;
4013         uint32_t dst_struct_id, src_struct_id;
4014
4015         CHECK(n_tokens == 3, EINVAL);
4016
4017         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4018         CHECK(fdst, EINVAL);
4019
4020         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
4021         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4022         if (fsrc) {
4023                 instr->type = INSTR_ALU_SHR;
4024                 if (dst[0] == 'h' && src[0] != 'h')
4025                         instr->type = INSTR_ALU_SHR_HM;
4026                 if (dst[0] != 'h' && src[0] == 'h')
4027                         instr->type = INSTR_ALU_SHR_MH;
4028                 if (dst[0] == 'h' && src[0] == 'h')
4029                         instr->type = INSTR_ALU_SHR_HH;
4030
4031                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4032                 instr->alu.dst.n_bits = fdst->n_bits;
4033                 instr->alu.dst.offset = fdst->offset / 8;
4034                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4035                 instr->alu.src.n_bits = fsrc->n_bits;
4036                 instr->alu.src.offset = fsrc->offset / 8;
4037                 return 0;
4038         }
4039
4040         /* SHR_MI, SHR_HI. */
4041         src_val = strtoull(src, &src, 0);
4042         CHECK(!src[0], EINVAL);
4043
4044         instr->type = INSTR_ALU_SHR_MI;
4045         if (dst[0] == 'h')
4046                 instr->type = INSTR_ALU_SHR_HI;
4047
4048         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4049         instr->alu.dst.n_bits = fdst->n_bits;
4050         instr->alu.dst.offset = fdst->offset / 8;
4051         instr->alu.src_val = src_val;
4052         return 0;
4053 }
4054
4055 static int
4056 instr_alu_and_translate(struct rte_swx_pipeline *p,
4057                         struct action *action,
4058                         char **tokens,
4059                         int n_tokens,
4060                         struct instruction *instr,
4061                         struct instruction_data *data __rte_unused)
4062 {
4063         char *dst = tokens[1], *src = tokens[2];
4064         struct field *fdst, *fsrc;
4065         uint64_t src_val;
4066         uint32_t dst_struct_id, src_struct_id;
4067
4068         CHECK(n_tokens == 3, EINVAL);
4069
4070         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4071         CHECK(fdst, EINVAL);
4072
4073         /* AND or AND_S. */
4074         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4075         if (fsrc) {
4076                 instr->type = INSTR_ALU_AND;
4077                 if ((dst[0] == 'h' && src[0] != 'h') ||
4078                     (dst[0] != 'h' && src[0] == 'h'))
4079                         instr->type = INSTR_ALU_AND_S;
4080
4081                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4082                 instr->alu.dst.n_bits = fdst->n_bits;
4083                 instr->alu.dst.offset = fdst->offset / 8;
4084                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4085                 instr->alu.src.n_bits = fsrc->n_bits;
4086                 instr->alu.src.offset = fsrc->offset / 8;
4087                 return 0;
4088         }
4089
4090         /* AND_I. */
4091         src_val = strtoull(src, &src, 0);
4092         CHECK(!src[0], EINVAL);
4093
4094         if (dst[0] == 'h')
4095                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4096
4097         instr->type = INSTR_ALU_AND_I;
4098         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4099         instr->alu.dst.n_bits = fdst->n_bits;
4100         instr->alu.dst.offset = fdst->offset / 8;
4101         instr->alu.src_val = src_val;
4102         return 0;
4103 }
4104
4105 static int
4106 instr_alu_or_translate(struct rte_swx_pipeline *p,
4107                        struct action *action,
4108                        char **tokens,
4109                        int n_tokens,
4110                        struct instruction *instr,
4111                        struct instruction_data *data __rte_unused)
4112 {
4113         char *dst = tokens[1], *src = tokens[2];
4114         struct field *fdst, *fsrc;
4115         uint64_t src_val;
4116         uint32_t dst_struct_id, src_struct_id;
4117
4118         CHECK(n_tokens == 3, EINVAL);
4119
4120         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4121         CHECK(fdst, EINVAL);
4122
4123         /* OR or OR_S. */
4124         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4125         if (fsrc) {
4126                 instr->type = INSTR_ALU_OR;
4127                 if ((dst[0] == 'h' && src[0] != 'h') ||
4128                     (dst[0] != 'h' && src[0] == 'h'))
4129                         instr->type = INSTR_ALU_OR_S;
4130
4131                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4132                 instr->alu.dst.n_bits = fdst->n_bits;
4133                 instr->alu.dst.offset = fdst->offset / 8;
4134                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4135                 instr->alu.src.n_bits = fsrc->n_bits;
4136                 instr->alu.src.offset = fsrc->offset / 8;
4137                 return 0;
4138         }
4139
4140         /* OR_I. */
4141         src_val = strtoull(src, &src, 0);
4142         CHECK(!src[0], EINVAL);
4143
4144         if (dst[0] == 'h')
4145                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4146
4147         instr->type = INSTR_ALU_OR_I;
4148         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4149         instr->alu.dst.n_bits = fdst->n_bits;
4150         instr->alu.dst.offset = fdst->offset / 8;
4151         instr->alu.src_val = src_val;
4152         return 0;
4153 }
4154
4155 static int
4156 instr_alu_xor_translate(struct rte_swx_pipeline *p,
4157                         struct action *action,
4158                         char **tokens,
4159                         int n_tokens,
4160                         struct instruction *instr,
4161                         struct instruction_data *data __rte_unused)
4162 {
4163         char *dst = tokens[1], *src = tokens[2];
4164         struct field *fdst, *fsrc;
4165         uint64_t src_val;
4166         uint32_t dst_struct_id, src_struct_id;
4167
4168         CHECK(n_tokens == 3, EINVAL);
4169
4170         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4171         CHECK(fdst, EINVAL);
4172
4173         /* XOR or XOR_S. */
4174         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4175         if (fsrc) {
4176                 instr->type = INSTR_ALU_XOR;
4177                 if ((dst[0] == 'h' && src[0] != 'h') ||
4178                     (dst[0] != 'h' && src[0] == 'h'))
4179                         instr->type = INSTR_ALU_XOR_S;
4180
4181                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4182                 instr->alu.dst.n_bits = fdst->n_bits;
4183                 instr->alu.dst.offset = fdst->offset / 8;
4184                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4185                 instr->alu.src.n_bits = fsrc->n_bits;
4186                 instr->alu.src.offset = fsrc->offset / 8;
4187                 return 0;
4188         }
4189
4190         /* XOR_I. */
4191         src_val = strtoull(src, &src, 0);
4192         CHECK(!src[0], EINVAL);
4193
4194         if (dst[0] == 'h')
4195                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4196
4197         instr->type = INSTR_ALU_XOR_I;
4198         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4199         instr->alu.dst.n_bits = fdst->n_bits;
4200         instr->alu.dst.offset = fdst->offset / 8;
4201         instr->alu.src_val = src_val;
4202         return 0;
4203 }
4204
4205 static inline void
4206 instr_alu_add_exec(struct rte_swx_pipeline *p)
4207 {
4208         struct thread *t = &p->threads[p->thread_id];
4209         struct instruction *ip = t->ip;
4210
4211         TRACE("[Thread %2u] add\n", p->thread_id);
4212
4213         /* Structs. */
4214         ALU(t, ip, +);
4215
4216         /* Thread. */
4217         thread_ip_inc(p);
4218 }
4219
4220 static inline void
4221 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
4222 {
4223         struct thread *t = &p->threads[p->thread_id];
4224         struct instruction *ip = t->ip;
4225
4226         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
4227
4228         /* Structs. */
4229         ALU_MH(t, ip, +);
4230
4231         /* Thread. */
4232         thread_ip_inc(p);
4233 }
4234
4235 static inline void
4236 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
4237 {
4238         struct thread *t = &p->threads[p->thread_id];
4239         struct instruction *ip = t->ip;
4240
4241         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
4242
4243         /* Structs. */
4244         ALU_HM(t, ip, +);
4245
4246         /* Thread. */
4247         thread_ip_inc(p);
4248 }
4249
4250 static inline void
4251 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
4252 {
4253         struct thread *t = &p->threads[p->thread_id];
4254         struct instruction *ip = t->ip;
4255
4256         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
4257
4258         /* Structs. */
4259         ALU_HH(t, ip, +);
4260
4261         /* Thread. */
4262         thread_ip_inc(p);
4263 }
4264
4265 static inline void
4266 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
4267 {
4268         struct thread *t = &p->threads[p->thread_id];
4269         struct instruction *ip = t->ip;
4270
4271         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
4272
4273         /* Structs. */
4274         ALU_MI(t, ip, +);
4275
4276         /* Thread. */
4277         thread_ip_inc(p);
4278 }
4279
4280 static inline void
4281 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
4282 {
4283         struct thread *t = &p->threads[p->thread_id];
4284         struct instruction *ip = t->ip;
4285
4286         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
4287
4288         /* Structs. */
4289         ALU_HI(t, ip, +);
4290
4291         /* Thread. */
4292         thread_ip_inc(p);
4293 }
4294
4295 static inline void
4296 instr_alu_sub_exec(struct rte_swx_pipeline *p)
4297 {
4298         struct thread *t = &p->threads[p->thread_id];
4299         struct instruction *ip = t->ip;
4300
4301         TRACE("[Thread %2u] sub\n", p->thread_id);
4302
4303         /* Structs. */
4304         ALU(t, ip, -);
4305
4306         /* Thread. */
4307         thread_ip_inc(p);
4308 }
4309
4310 static inline void
4311 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
4312 {
4313         struct thread *t = &p->threads[p->thread_id];
4314         struct instruction *ip = t->ip;
4315
4316         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
4317
4318         /* Structs. */
4319         ALU_MH(t, ip, -);
4320
4321         /* Thread. */
4322         thread_ip_inc(p);
4323 }
4324
4325 static inline void
4326 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4327 {
4328         struct thread *t = &p->threads[p->thread_id];
4329         struct instruction *ip = t->ip;
4330
4331         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4332
4333         /* Structs. */
4334         ALU_HM(t, ip, -);
4335
4336         /* Thread. */
4337         thread_ip_inc(p);
4338 }
4339
4340 static inline void
4341 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4342 {
4343         struct thread *t = &p->threads[p->thread_id];
4344         struct instruction *ip = t->ip;
4345
4346         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4347
4348         /* Structs. */
4349         ALU_HH(t, ip, -);
4350
4351         /* Thread. */
4352         thread_ip_inc(p);
4353 }
4354
4355 static inline void
4356 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4357 {
4358         struct thread *t = &p->threads[p->thread_id];
4359         struct instruction *ip = t->ip;
4360
4361         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4362
4363         /* Structs. */
4364         ALU_MI(t, ip, -);
4365
4366         /* Thread. */
4367         thread_ip_inc(p);
4368 }
4369
4370 static inline void
4371 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4372 {
4373         struct thread *t = &p->threads[p->thread_id];
4374         struct instruction *ip = t->ip;
4375
4376         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4377
4378         /* Structs. */
4379         ALU_HI(t, ip, -);
4380
4381         /* Thread. */
4382         thread_ip_inc(p);
4383 }
4384
4385 static inline void
4386 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4387 {
4388         struct thread *t = &p->threads[p->thread_id];
4389         struct instruction *ip = t->ip;
4390
4391         TRACE("[Thread %2u] shl\n", p->thread_id);
4392
4393         /* Structs. */
4394         ALU(t, ip, <<);
4395
4396         /* Thread. */
4397         thread_ip_inc(p);
4398 }
4399
4400 static inline void
4401 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4402 {
4403         struct thread *t = &p->threads[p->thread_id];
4404         struct instruction *ip = t->ip;
4405
4406         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4407
4408         /* Structs. */
4409         ALU_MH(t, ip, <<);
4410
4411         /* Thread. */
4412         thread_ip_inc(p);
4413 }
4414
4415 static inline void
4416 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4417 {
4418         struct thread *t = &p->threads[p->thread_id];
4419         struct instruction *ip = t->ip;
4420
4421         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4422
4423         /* Structs. */
4424         ALU_HM(t, ip, <<);
4425
4426         /* Thread. */
4427         thread_ip_inc(p);
4428 }
4429
4430 static inline void
4431 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4432 {
4433         struct thread *t = &p->threads[p->thread_id];
4434         struct instruction *ip = t->ip;
4435
4436         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4437
4438         /* Structs. */
4439         ALU_HH(t, ip, <<);
4440
4441         /* Thread. */
4442         thread_ip_inc(p);
4443 }
4444
4445 static inline void
4446 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4447 {
4448         struct thread *t = &p->threads[p->thread_id];
4449         struct instruction *ip = t->ip;
4450
4451         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4452
4453         /* Structs. */
4454         ALU_MI(t, ip, <<);
4455
4456         /* Thread. */
4457         thread_ip_inc(p);
4458 }
4459
4460 static inline void
4461 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4462 {
4463         struct thread *t = &p->threads[p->thread_id];
4464         struct instruction *ip = t->ip;
4465
4466         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4467
4468         /* Structs. */
4469         ALU_HI(t, ip, <<);
4470
4471         /* Thread. */
4472         thread_ip_inc(p);
4473 }
4474
4475 static inline void
4476 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4477 {
4478         struct thread *t = &p->threads[p->thread_id];
4479         struct instruction *ip = t->ip;
4480
4481         TRACE("[Thread %2u] shr\n", p->thread_id);
4482
4483         /* Structs. */
4484         ALU(t, ip, >>);
4485
4486         /* Thread. */
4487         thread_ip_inc(p);
4488 }
4489
4490 static inline void
4491 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4492 {
4493         struct thread *t = &p->threads[p->thread_id];
4494         struct instruction *ip = t->ip;
4495
4496         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4497
4498         /* Structs. */
4499         ALU_MH(t, ip, >>);
4500
4501         /* Thread. */
4502         thread_ip_inc(p);
4503 }
4504
4505 static inline void
4506 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4507 {
4508         struct thread *t = &p->threads[p->thread_id];
4509         struct instruction *ip = t->ip;
4510
4511         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4512
4513         /* Structs. */
4514         ALU_HM(t, ip, >>);
4515
4516         /* Thread. */
4517         thread_ip_inc(p);
4518 }
4519
4520 static inline void
4521 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4522 {
4523         struct thread *t = &p->threads[p->thread_id];
4524         struct instruction *ip = t->ip;
4525
4526         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4527
4528         /* Structs. */
4529         ALU_HH(t, ip, >>);
4530
4531         /* Thread. */
4532         thread_ip_inc(p);
4533 }
4534
4535 static inline void
4536 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4537 {
4538         struct thread *t = &p->threads[p->thread_id];
4539         struct instruction *ip = t->ip;
4540
4541         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4542
4543         /* Structs. */
4544         ALU_MI(t, ip, >>);
4545
4546         /* Thread. */
4547         thread_ip_inc(p);
4548 }
4549
4550 static inline void
4551 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4552 {
4553         struct thread *t = &p->threads[p->thread_id];
4554         struct instruction *ip = t->ip;
4555
4556         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4557
4558         /* Structs. */
4559         ALU_HI(t, ip, >>);
4560
4561         /* Thread. */
4562         thread_ip_inc(p);
4563 }
4564
4565 static inline void
4566 instr_alu_and_exec(struct rte_swx_pipeline *p)
4567 {
4568         struct thread *t = &p->threads[p->thread_id];
4569         struct instruction *ip = t->ip;
4570
4571         TRACE("[Thread %2u] and\n", p->thread_id);
4572
4573         /* Structs. */
4574         ALU(t, ip, &);
4575
4576         /* Thread. */
4577         thread_ip_inc(p);
4578 }
4579
4580 static inline void
4581 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
4582 {
4583         struct thread *t = &p->threads[p->thread_id];
4584         struct instruction *ip = t->ip;
4585
4586         TRACE("[Thread %2u] and (s)\n", p->thread_id);
4587
4588         /* Structs. */
4589         ALU_S(t, ip, &);
4590
4591         /* Thread. */
4592         thread_ip_inc(p);
4593 }
4594
4595 static inline void
4596 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4597 {
4598         struct thread *t = &p->threads[p->thread_id];
4599         struct instruction *ip = t->ip;
4600
4601         TRACE("[Thread %2u] and (i)\n", p->thread_id);
4602
4603         /* Structs. */
4604         ALU_I(t, ip, &);
4605
4606         /* Thread. */
4607         thread_ip_inc(p);
4608 }
4609
4610 static inline void
4611 instr_alu_or_exec(struct rte_swx_pipeline *p)
4612 {
4613         struct thread *t = &p->threads[p->thread_id];
4614         struct instruction *ip = t->ip;
4615
4616         TRACE("[Thread %2u] or\n", p->thread_id);
4617
4618         /* Structs. */
4619         ALU(t, ip, |);
4620
4621         /* Thread. */
4622         thread_ip_inc(p);
4623 }
4624
4625 static inline void
4626 instr_alu_or_s_exec(struct rte_swx_pipeline *p)
4627 {
4628         struct thread *t = &p->threads[p->thread_id];
4629         struct instruction *ip = t->ip;
4630
4631         TRACE("[Thread %2u] or (s)\n", p->thread_id);
4632
4633         /* Structs. */
4634         ALU_S(t, ip, |);
4635
4636         /* Thread. */
4637         thread_ip_inc(p);
4638 }
4639
4640 static inline void
4641 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4642 {
4643         struct thread *t = &p->threads[p->thread_id];
4644         struct instruction *ip = t->ip;
4645
4646         TRACE("[Thread %2u] or (i)\n", p->thread_id);
4647
4648         /* Structs. */
4649         ALU_I(t, ip, |);
4650
4651         /* Thread. */
4652         thread_ip_inc(p);
4653 }
4654
4655 static inline void
4656 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4657 {
4658         struct thread *t = &p->threads[p->thread_id];
4659         struct instruction *ip = t->ip;
4660
4661         TRACE("[Thread %2u] xor\n", p->thread_id);
4662
4663         /* Structs. */
4664         ALU(t, ip, ^);
4665
4666         /* Thread. */
4667         thread_ip_inc(p);
4668 }
4669
4670 static inline void
4671 instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
4672 {
4673         struct thread *t = &p->threads[p->thread_id];
4674         struct instruction *ip = t->ip;
4675
4676         TRACE("[Thread %2u] xor (s)\n", p->thread_id);
4677
4678         /* Structs. */
4679         ALU_S(t, ip, ^);
4680
4681         /* Thread. */
4682         thread_ip_inc(p);
4683 }
4684
4685 static inline void
4686 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4687 {
4688         struct thread *t = &p->threads[p->thread_id];
4689         struct instruction *ip = t->ip;
4690
4691         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
4692
4693         /* Structs. */
4694         ALU_I(t, ip, ^);
4695
4696         /* Thread. */
4697         thread_ip_inc(p);
4698 }
4699
4700 static inline void
4701 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4702 {
4703         struct thread *t = &p->threads[p->thread_id];
4704         struct instruction *ip = t->ip;
4705         uint8_t *dst_struct, *src_struct;
4706         uint16_t *dst16_ptr, dst;
4707         uint64_t *src64_ptr, src64, src64_mask, src;
4708         uint64_t r;
4709
4710         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
4711
4712         /* Structs. */
4713         dst_struct = t->structs[ip->alu.dst.struct_id];
4714         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4715         dst = *dst16_ptr;
4716
4717         src_struct = t->structs[ip->alu.src.struct_id];
4718         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4719         src64 = *src64_ptr;
4720         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4721         src = src64 & src64_mask;
4722
4723         r = dst;
4724         r = ~r & 0xFFFF;
4725
4726         /* The first input (r) is a 16-bit number. The second and the third
4727          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
4728          * three numbers (output r) is a 34-bit number.
4729          */
4730         r += (src >> 32) + (src & 0xFFFFFFFF);
4731
4732         /* The first input is a 16-bit number. The second input is an 18-bit
4733          * number. In the worst case scenario, the sum of the two numbers is a
4734          * 19-bit number.
4735          */
4736         r = (r & 0xFFFF) + (r >> 16);
4737
4738         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4739          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
4740          */
4741         r = (r & 0xFFFF) + (r >> 16);
4742
4743         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4744          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4745          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
4746          * therefore the output r is always a 16-bit number.
4747          */
4748         r = (r & 0xFFFF) + (r >> 16);
4749
4750         r = ~r & 0xFFFF;
4751         r = r ? r : 0xFFFF;
4752
4753         *dst16_ptr = (uint16_t)r;
4754
4755         /* Thread. */
4756         thread_ip_inc(p);
4757 }
4758
4759 static inline void
4760 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4761 {
4762         struct thread *t = &p->threads[p->thread_id];
4763         struct instruction *ip = t->ip;
4764         uint8_t *dst_struct, *src_struct;
4765         uint16_t *dst16_ptr, dst;
4766         uint64_t *src64_ptr, src64, src64_mask, src;
4767         uint64_t r;
4768
4769         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
4770
4771         /* Structs. */
4772         dst_struct = t->structs[ip->alu.dst.struct_id];
4773         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4774         dst = *dst16_ptr;
4775
4776         src_struct = t->structs[ip->alu.src.struct_id];
4777         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4778         src64 = *src64_ptr;
4779         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4780         src = src64 & src64_mask;
4781
4782         r = dst;
4783         r = ~r & 0xFFFF;
4784
4785         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
4786          * the following sequence of operations in 2's complement arithmetic:
4787          *    a '- b = (a - b) % 0xFFFF.
4788          *
4789          * In order to prevent an underflow for the below subtraction, in which
4790          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
4791          * minuend), we first add a multiple of the 0xFFFF modulus to the
4792          * minuend. The number we add to the minuend needs to be a 34-bit number
4793          * or higher, so for readability reasons we picked the 36-bit multiple.
4794          * We are effectively turning the 16-bit minuend into a 36-bit number:
4795          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
4796          */
4797         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
4798
4799         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
4800          * result (the output r) is a 36-bit number.
4801          */
4802         r -= (src >> 32) + (src & 0xFFFFFFFF);
4803
4804         /* The first input is a 16-bit number. The second input is a 20-bit
4805          * number. Their sum is a 21-bit number.
4806          */
4807         r = (r & 0xFFFF) + (r >> 16);
4808
4809         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4810          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
4811          */
4812         r = (r & 0xFFFF) + (r >> 16);
4813
4814         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4815          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4816          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4817          * generated, therefore the output r is always a 16-bit number.
4818          */
4819         r = (r & 0xFFFF) + (r >> 16);
4820
4821         r = ~r & 0xFFFF;
4822         r = r ? r : 0xFFFF;
4823
4824         *dst16_ptr = (uint16_t)r;
4825
4826         /* Thread. */
4827         thread_ip_inc(p);
4828 }
4829
4830 static inline void
4831 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4832 {
4833         struct thread *t = &p->threads[p->thread_id];
4834         struct instruction *ip = t->ip;
4835         uint8_t *dst_struct, *src_struct;
4836         uint16_t *dst16_ptr;
4837         uint32_t *src32_ptr;
4838         uint64_t r0, r1;
4839
4840         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
4841
4842         /* Structs. */
4843         dst_struct = t->structs[ip->alu.dst.struct_id];
4844         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4845
4846         src_struct = t->structs[ip->alu.src.struct_id];
4847         src32_ptr = (uint32_t *)&src_struct[0];
4848
4849         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
4850         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
4851         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
4852         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
4853         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
4854
4855         /* The first input is a 16-bit number. The second input is a 19-bit
4856          * number. Their sum is a 20-bit number.
4857          */
4858         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4859
4860         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4861          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
4862          */
4863         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4864
4865         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4866          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4867          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
4868          * generated, therefore the output r is always a 16-bit number.
4869          */
4870         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4871
4872         r0 = ~r0 & 0xFFFF;
4873         r0 = r0 ? r0 : 0xFFFF;
4874
4875         *dst16_ptr = (uint16_t)r0;
4876
4877         /* Thread. */
4878         thread_ip_inc(p);
4879 }
4880
4881 static inline void
4882 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4883 {
4884         struct thread *t = &p->threads[p->thread_id];
4885         struct instruction *ip = t->ip;
4886         uint8_t *dst_struct, *src_struct;
4887         uint16_t *dst16_ptr;
4888         uint32_t *src32_ptr;
4889         uint64_t r = 0;
4890         uint32_t i;
4891
4892         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
4893
4894         /* Structs. */
4895         dst_struct = t->structs[ip->alu.dst.struct_id];
4896         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4897
4898         src_struct = t->structs[ip->alu.src.struct_id];
4899         src32_ptr = (uint32_t *)&src_struct[0];
4900
4901         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
4902          * Therefore, in the worst case scenario, a 35-bit number is added to a
4903          * 16-bit number (the input r), so the output r is 36-bit number.
4904          */
4905         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
4906                 r += *src32_ptr;
4907
4908         /* The first input is a 16-bit number. The second input is a 20-bit
4909          * number. Their sum is a 21-bit number.
4910          */
4911         r = (r & 0xFFFF) + (r >> 16);
4912
4913         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4914          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
4915          */
4916         r = (r & 0xFFFF) + (r >> 16);
4917
4918         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4919          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4920          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4921          * generated, therefore the output r is always a 16-bit number.
4922          */
4923         r = (r & 0xFFFF) + (r >> 16);
4924
4925         r = ~r & 0xFFFF;
4926         r = r ? r : 0xFFFF;
4927
4928         *dst16_ptr = (uint16_t)r;
4929
4930         /* Thread. */
4931         thread_ip_inc(p);
4932 }
4933
4934 /*
4935  * Register array.
4936  */
4937 static struct regarray *
4938 regarray_find(struct rte_swx_pipeline *p, const char *name);
4939
4940 static int
4941 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4942                       struct action *action,
4943                       char **tokens,
4944                       int n_tokens,
4945                       struct instruction *instr,
4946                       struct instruction_data *data __rte_unused)
4947 {
4948         char *regarray = tokens[1], *idx = tokens[2];
4949         struct regarray *r;
4950         struct field *fidx;
4951         uint32_t idx_struct_id, idx_val;
4952
4953         CHECK(n_tokens == 3, EINVAL);
4954
4955         r = regarray_find(p, regarray);
4956         CHECK(r, EINVAL);
4957
4958         /* REGPREFETCH_RH, REGPREFETCH_RM. */
4959         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4960         if (fidx) {
4961                 instr->type = INSTR_REGPREFETCH_RM;
4962                 if (idx[0] == 'h')
4963                         instr->type = INSTR_REGPREFETCH_RH;
4964
4965                 instr->regarray.regarray_id = r->id;
4966                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4967                 instr->regarray.idx.n_bits = fidx->n_bits;
4968                 instr->regarray.idx.offset = fidx->offset / 8;
4969                 instr->regarray.dstsrc_val = 0; /* Unused. */
4970                 return 0;
4971         }
4972
4973         /* REGPREFETCH_RI. */
4974         idx_val = strtoul(idx, &idx, 0);
4975         CHECK(!idx[0], EINVAL);
4976
4977         instr->type = INSTR_REGPREFETCH_RI;
4978         instr->regarray.regarray_id = r->id;
4979         instr->regarray.idx_val = idx_val;
4980         instr->regarray.dstsrc_val = 0; /* Unused. */
4981         return 0;
4982 }
4983
4984 static int
4985 instr_regrd_translate(struct rte_swx_pipeline *p,
4986                       struct action *action,
4987                       char **tokens,
4988                       int n_tokens,
4989                       struct instruction *instr,
4990                       struct instruction_data *data __rte_unused)
4991 {
4992         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4993         struct regarray *r;
4994         struct field *fdst, *fidx;
4995         uint32_t dst_struct_id, idx_struct_id, idx_val;
4996
4997         CHECK(n_tokens == 4, EINVAL);
4998
4999         r = regarray_find(p, regarray);
5000         CHECK(r, EINVAL);
5001
5002         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
5003         CHECK(fdst, EINVAL);
5004
5005         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
5006         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5007         if (fidx) {
5008                 instr->type = INSTR_REGRD_MRM;
5009                 if (dst[0] == 'h' && idx[0] != 'h')
5010                         instr->type = INSTR_REGRD_HRM;
5011                 if (dst[0] != 'h' && idx[0] == 'h')
5012                         instr->type = INSTR_REGRD_MRH;
5013                 if (dst[0] == 'h' && idx[0] == 'h')
5014                         instr->type = INSTR_REGRD_HRH;
5015
5016                 instr->regarray.regarray_id = r->id;
5017                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5018                 instr->regarray.idx.n_bits = fidx->n_bits;
5019                 instr->regarray.idx.offset = fidx->offset / 8;
5020                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5021                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
5022                 instr->regarray.dstsrc.offset = fdst->offset / 8;
5023                 return 0;
5024         }
5025
5026         /* REGRD_MRI, REGRD_HRI. */
5027         idx_val = strtoul(idx, &idx, 0);
5028         CHECK(!idx[0], EINVAL);
5029
5030         instr->type = INSTR_REGRD_MRI;
5031         if (dst[0] == 'h')
5032                 instr->type = INSTR_REGRD_HRI;
5033
5034         instr->regarray.regarray_id = r->id;
5035         instr->regarray.idx_val = idx_val;
5036         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5037         instr->regarray.dstsrc.n_bits = fdst->n_bits;
5038         instr->regarray.dstsrc.offset = fdst->offset / 8;
5039         return 0;
5040 }
5041
5042 static int
5043 instr_regwr_translate(struct rte_swx_pipeline *p,
5044                       struct action *action,
5045                       char **tokens,
5046                       int n_tokens,
5047                       struct instruction *instr,
5048                       struct instruction_data *data __rte_unused)
5049 {
5050         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5051         struct regarray *r;
5052         struct field *fidx, *fsrc;
5053         uint64_t src_val;
5054         uint32_t idx_struct_id, idx_val, src_struct_id;
5055
5056         CHECK(n_tokens == 4, EINVAL);
5057
5058         r = regarray_find(p, regarray);
5059         CHECK(r, EINVAL);
5060
5061         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
5062         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5063         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5064         if (fidx && fsrc) {
5065                 instr->type = INSTR_REGWR_RMM;
5066                 if (idx[0] == 'h' && src[0] != 'h')
5067                         instr->type = INSTR_REGWR_RHM;
5068                 if (idx[0] != 'h' && src[0] == 'h')
5069                         instr->type = INSTR_REGWR_RMH;
5070                 if (idx[0] == 'h' && src[0] == 'h')
5071                         instr->type = INSTR_REGWR_RHH;
5072
5073                 instr->regarray.regarray_id = r->id;
5074                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5075                 instr->regarray.idx.n_bits = fidx->n_bits;
5076                 instr->regarray.idx.offset = fidx->offset / 8;
5077                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5078                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5079                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5080                 return 0;
5081         }
5082
5083         /* REGWR_RHI, REGWR_RMI. */
5084         if (fidx && !fsrc) {
5085                 src_val = strtoull(src, &src, 0);
5086                 CHECK(!src[0], EINVAL);
5087
5088                 instr->type = INSTR_REGWR_RMI;
5089                 if (idx[0] == 'h')
5090                         instr->type = INSTR_REGWR_RHI;
5091
5092                 instr->regarray.regarray_id = r->id;
5093                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5094                 instr->regarray.idx.n_bits = fidx->n_bits;
5095                 instr->regarray.idx.offset = fidx->offset / 8;
5096                 instr->regarray.dstsrc_val = src_val;
5097                 return 0;
5098         }
5099
5100         /* REGWR_RIH, REGWR_RIM. */
5101         if (!fidx && fsrc) {
5102                 idx_val = strtoul(idx, &idx, 0);
5103                 CHECK(!idx[0], EINVAL);
5104
5105                 instr->type = INSTR_REGWR_RIM;
5106                 if (src[0] == 'h')
5107                         instr->type = INSTR_REGWR_RIH;
5108
5109                 instr->regarray.regarray_id = r->id;
5110                 instr->regarray.idx_val = idx_val;
5111                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5112                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5113                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5114                 return 0;
5115         }
5116
5117         /* REGWR_RII. */
5118         src_val = strtoull(src, &src, 0);
5119         CHECK(!src[0], EINVAL);
5120
5121         idx_val = strtoul(idx, &idx, 0);
5122         CHECK(!idx[0], EINVAL);
5123
5124         instr->type = INSTR_REGWR_RII;
5125         instr->regarray.idx_val = idx_val;
5126         instr->regarray.dstsrc_val = src_val;
5127
5128         return 0;
5129 }
5130
5131 static int
5132 instr_regadd_translate(struct rte_swx_pipeline *p,
5133                        struct action *action,
5134                        char **tokens,
5135                        int n_tokens,
5136                        struct instruction *instr,
5137                        struct instruction_data *data __rte_unused)
5138 {
5139         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5140         struct regarray *r;
5141         struct field *fidx, *fsrc;
5142         uint64_t src_val;
5143         uint32_t idx_struct_id, idx_val, src_struct_id;
5144
5145         CHECK(n_tokens == 4, EINVAL);
5146
5147         r = regarray_find(p, regarray);
5148         CHECK(r, EINVAL);
5149
5150         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
5151         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5152         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5153         if (fidx && fsrc) {
5154                 instr->type = INSTR_REGADD_RMM;
5155                 if (idx[0] == 'h' && src[0] != 'h')
5156                         instr->type = INSTR_REGADD_RHM;
5157                 if (idx[0] != 'h' && src[0] == 'h')
5158                         instr->type = INSTR_REGADD_RMH;
5159                 if (idx[0] == 'h' && src[0] == 'h')
5160                         instr->type = INSTR_REGADD_RHH;
5161
5162                 instr->regarray.regarray_id = r->id;
5163                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5164                 instr->regarray.idx.n_bits = fidx->n_bits;
5165                 instr->regarray.idx.offset = fidx->offset / 8;
5166                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5167                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5168                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5169                 return 0;
5170         }
5171
5172         /* REGADD_RHI, REGADD_RMI. */
5173         if (fidx && !fsrc) {
5174                 src_val = strtoull(src, &src, 0);
5175                 CHECK(!src[0], EINVAL);
5176
5177                 instr->type = INSTR_REGADD_RMI;
5178                 if (idx[0] == 'h')
5179                         instr->type = INSTR_REGADD_RHI;
5180
5181                 instr->regarray.regarray_id = r->id;
5182                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5183                 instr->regarray.idx.n_bits = fidx->n_bits;
5184                 instr->regarray.idx.offset = fidx->offset / 8;
5185                 instr->regarray.dstsrc_val = src_val;
5186                 return 0;
5187         }
5188
5189         /* REGADD_RIH, REGADD_RIM. */
5190         if (!fidx && fsrc) {
5191                 idx_val = strtoul(idx, &idx, 0);
5192                 CHECK(!idx[0], EINVAL);
5193
5194                 instr->type = INSTR_REGADD_RIM;
5195                 if (src[0] == 'h')
5196                         instr->type = INSTR_REGADD_RIH;
5197
5198                 instr->regarray.regarray_id = r->id;
5199                 instr->regarray.idx_val = idx_val;
5200                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5201                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5202                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5203                 return 0;
5204         }
5205
5206         /* REGADD_RII. */
5207         src_val = strtoull(src, &src, 0);
5208         CHECK(!src[0], EINVAL);
5209
5210         idx_val = strtoul(idx, &idx, 0);
5211         CHECK(!idx[0], EINVAL);
5212
5213         instr->type = INSTR_REGADD_RII;
5214         instr->regarray.idx_val = idx_val;
5215         instr->regarray.dstsrc_val = src_val;
5216         return 0;
5217 }
5218
5219 static inline uint64_t *
5220 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
5221 {
5222         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5223         return r->regarray;
5224 }
5225
5226 static inline uint64_t
5227 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5228 {
5229         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5230
5231         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5232         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5233         uint64_t idx64 = *idx64_ptr;
5234         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
5235         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5236
5237         return idx;
5238 }
5239
5240 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5241
5242 static inline uint64_t
5243 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5244 {
5245         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5246
5247         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5248         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5249         uint64_t idx64 = *idx64_ptr;
5250         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
5251
5252         return idx;
5253 }
5254
5255 #else
5256
5257 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
5258
5259 #endif
5260
5261 static inline uint64_t
5262 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5263 {
5264         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5265
5266         uint64_t idx = ip->regarray.idx_val & r->size_mask;
5267
5268         return idx;
5269 }
5270
5271 static inline uint64_t
5272 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
5273 {
5274         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5275         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5276         uint64_t src64 = *src64_ptr;
5277         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5278         uint64_t src = src64 & src64_mask;
5279
5280         return src;
5281 }
5282
5283 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5284
5285 static inline uint64_t
5286 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
5287 {
5288         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5289         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5290         uint64_t src64 = *src64_ptr;
5291         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
5292
5293         return src;
5294 }
5295
5296 #else
5297
5298 #define instr_regarray_src_nbo instr_regarray_src_hbo
5299
5300 #endif
5301
5302 static inline void
5303 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5304 {
5305         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5306         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5307         uint64_t dst64 = *dst64_ptr;
5308         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5309
5310         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5311
5312 }
5313
5314 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5315
5316 static inline void
5317 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5318 {
5319         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5320         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5321         uint64_t dst64 = *dst64_ptr;
5322         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5323
5324         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
5325         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5326 }
5327
5328 #else
5329
5330 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
5331
5332 #endif
5333
5334 static inline void
5335 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
5336 {
5337         struct thread *t = &p->threads[p->thread_id];
5338         struct instruction *ip = t->ip;
5339         uint64_t *regarray, idx;
5340
5341         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
5342
5343         /* Structs. */
5344         regarray = instr_regarray_regarray(p, ip);
5345         idx = instr_regarray_idx_nbo(p, t, ip);
5346         rte_prefetch0(&regarray[idx]);
5347
5348         /* Thread. */
5349         thread_ip_inc(p);
5350 }
5351
5352 static inline void
5353 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
5354 {
5355         struct thread *t = &p->threads[p->thread_id];
5356         struct instruction *ip = t->ip;
5357         uint64_t *regarray, idx;
5358
5359         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
5360
5361         /* Structs. */
5362         regarray = instr_regarray_regarray(p, ip);
5363         idx = instr_regarray_idx_hbo(p, t, ip);
5364         rte_prefetch0(&regarray[idx]);
5365
5366         /* Thread. */
5367         thread_ip_inc(p);
5368 }
5369
5370 static inline void
5371 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
5372 {
5373         struct thread *t = &p->threads[p->thread_id];
5374         struct instruction *ip = t->ip;
5375         uint64_t *regarray, idx;
5376
5377         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
5378
5379         /* Structs. */
5380         regarray = instr_regarray_regarray(p, ip);
5381         idx = instr_regarray_idx_imm(p, ip);
5382         rte_prefetch0(&regarray[idx]);
5383
5384         /* Thread. */
5385         thread_ip_inc(p);
5386 }
5387
5388 static inline void
5389 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
5390 {
5391         struct thread *t = &p->threads[p->thread_id];
5392         struct instruction *ip = t->ip;
5393         uint64_t *regarray, idx;
5394
5395         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
5396
5397         /* Structs. */
5398         regarray = instr_regarray_regarray(p, ip);
5399         idx = instr_regarray_idx_nbo(p, t, ip);
5400         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5401
5402         /* Thread. */
5403         thread_ip_inc(p);
5404 }
5405
5406 static inline void
5407 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
5408 {
5409         struct thread *t = &p->threads[p->thread_id];
5410         struct instruction *ip = t->ip;
5411         uint64_t *regarray, idx;
5412
5413         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
5414
5415         /* Structs. */
5416         regarray = instr_regarray_regarray(p, ip);
5417         idx = instr_regarray_idx_hbo(p, t, ip);
5418         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5419
5420         /* Thread. */
5421         thread_ip_inc(p);
5422 }
5423
5424 static inline void
5425 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
5426 {
5427         struct thread *t = &p->threads[p->thread_id];
5428         struct instruction *ip = t->ip;
5429         uint64_t *regarray, idx;
5430
5431         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
5432
5433         /* Structs. */
5434         regarray = instr_regarray_regarray(p, ip);
5435         idx = instr_regarray_idx_nbo(p, t, ip);
5436         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5437
5438         /* Thread. */
5439         thread_ip_inc(p);
5440 }
5441
5442 static inline void
5443 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
5444 {
5445         struct thread *t = &p->threads[p->thread_id];
5446         struct instruction *ip = t->ip;
5447         uint64_t *regarray, idx;
5448
5449         /* Structs. */
5450         regarray = instr_regarray_regarray(p, ip);
5451         idx = instr_regarray_idx_hbo(p, t, ip);
5452         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5453
5454         /* Thread. */
5455         thread_ip_inc(p);
5456 }
5457
5458 static inline void
5459 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
5460 {
5461         struct thread *t = &p->threads[p->thread_id];
5462         struct instruction *ip = t->ip;
5463         uint64_t *regarray, idx;
5464
5465         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
5466
5467         /* Structs. */
5468         regarray = instr_regarray_regarray(p, ip);
5469         idx = instr_regarray_idx_imm(p, ip);
5470         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5471
5472         /* Thread. */
5473         thread_ip_inc(p);
5474 }
5475
5476 static inline void
5477 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
5478 {
5479         struct thread *t = &p->threads[p->thread_id];
5480         struct instruction *ip = t->ip;
5481         uint64_t *regarray, idx;
5482
5483         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
5484
5485         /* Structs. */
5486         regarray = instr_regarray_regarray(p, ip);
5487         idx = instr_regarray_idx_imm(p, ip);
5488         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5489
5490         /* Thread. */
5491         thread_ip_inc(p);
5492 }
5493
5494 static inline void
5495 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
5496 {
5497         struct thread *t = &p->threads[p->thread_id];
5498         struct instruction *ip = t->ip;
5499         uint64_t *regarray, idx, src;
5500
5501         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
5502
5503         /* Structs. */
5504         regarray = instr_regarray_regarray(p, ip);
5505         idx = instr_regarray_idx_nbo(p, t, ip);
5506         src = instr_regarray_src_nbo(t, ip);
5507         regarray[idx] = src;
5508
5509         /* Thread. */
5510         thread_ip_inc(p);
5511 }
5512
5513 static inline void
5514 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
5515 {
5516         struct thread *t = &p->threads[p->thread_id];
5517         struct instruction *ip = t->ip;
5518         uint64_t *regarray, idx, src;
5519
5520         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
5521
5522         /* Structs. */
5523         regarray = instr_regarray_regarray(p, ip);
5524         idx = instr_regarray_idx_nbo(p, t, ip);
5525         src = instr_regarray_src_hbo(t, ip);
5526         regarray[idx] = src;
5527
5528         /* Thread. */
5529         thread_ip_inc(p);
5530 }
5531
5532 static inline void
5533 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
5534 {
5535         struct thread *t = &p->threads[p->thread_id];
5536         struct instruction *ip = t->ip;
5537         uint64_t *regarray, idx, src;
5538
5539         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
5540
5541         /* Structs. */
5542         regarray = instr_regarray_regarray(p, ip);
5543         idx = instr_regarray_idx_hbo(p, t, ip);
5544         src = instr_regarray_src_nbo(t, ip);
5545         regarray[idx] = src;
5546
5547         /* Thread. */
5548         thread_ip_inc(p);
5549 }
5550
5551 static inline void
5552 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
5553 {
5554         struct thread *t = &p->threads[p->thread_id];
5555         struct instruction *ip = t->ip;
5556         uint64_t *regarray, idx, src;
5557
5558         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
5559
5560         /* Structs. */
5561         regarray = instr_regarray_regarray(p, ip);
5562         idx = instr_regarray_idx_hbo(p, t, ip);
5563         src = instr_regarray_src_hbo(t, ip);
5564         regarray[idx] = src;
5565
5566         /* Thread. */
5567         thread_ip_inc(p);
5568 }
5569
5570 static inline void
5571 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
5572 {
5573         struct thread *t = &p->threads[p->thread_id];
5574         struct instruction *ip = t->ip;
5575         uint64_t *regarray, idx, src;
5576
5577         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
5578
5579         /* Structs. */
5580         regarray = instr_regarray_regarray(p, ip);
5581         idx = instr_regarray_idx_nbo(p, t, ip);
5582         src = ip->regarray.dstsrc_val;
5583         regarray[idx] = src;
5584
5585         /* Thread. */
5586         thread_ip_inc(p);
5587 }
5588
5589 static inline void
5590 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
5591 {
5592         struct thread *t = &p->threads[p->thread_id];
5593         struct instruction *ip = t->ip;
5594         uint64_t *regarray, idx, src;
5595
5596         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
5597
5598         /* Structs. */
5599         regarray = instr_regarray_regarray(p, ip);
5600         idx = instr_regarray_idx_hbo(p, t, ip);
5601         src = ip->regarray.dstsrc_val;
5602         regarray[idx] = src;
5603
5604         /* Thread. */
5605         thread_ip_inc(p);
5606 }
5607
5608 static inline void
5609 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
5610 {
5611         struct thread *t = &p->threads[p->thread_id];
5612         struct instruction *ip = t->ip;
5613         uint64_t *regarray, idx, src;
5614
5615         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
5616
5617         /* Structs. */
5618         regarray = instr_regarray_regarray(p, ip);
5619         idx = instr_regarray_idx_imm(p, ip);
5620         src = instr_regarray_src_nbo(t, ip);
5621         regarray[idx] = src;
5622
5623         /* Thread. */
5624         thread_ip_inc(p);
5625 }
5626
5627 static inline void
5628 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
5629 {
5630         struct thread *t = &p->threads[p->thread_id];
5631         struct instruction *ip = t->ip;
5632         uint64_t *regarray, idx, src;
5633
5634         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
5635
5636         /* Structs. */
5637         regarray = instr_regarray_regarray(p, ip);
5638         idx = instr_regarray_idx_imm(p, ip);
5639         src = instr_regarray_src_hbo(t, ip);
5640         regarray[idx] = src;
5641
5642         /* Thread. */
5643         thread_ip_inc(p);
5644 }
5645
5646 static inline void
5647 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
5648 {
5649         struct thread *t = &p->threads[p->thread_id];
5650         struct instruction *ip = t->ip;
5651         uint64_t *regarray, idx, src;
5652
5653         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
5654
5655         /* Structs. */
5656         regarray = instr_regarray_regarray(p, ip);
5657         idx = instr_regarray_idx_imm(p, ip);
5658         src = ip->regarray.dstsrc_val;
5659         regarray[idx] = src;
5660
5661         /* Thread. */
5662         thread_ip_inc(p);
5663 }
5664
5665 static inline void
5666 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
5667 {
5668         struct thread *t = &p->threads[p->thread_id];
5669         struct instruction *ip = t->ip;
5670         uint64_t *regarray, idx, src;
5671
5672         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
5673
5674         /* Structs. */
5675         regarray = instr_regarray_regarray(p, ip);
5676         idx = instr_regarray_idx_nbo(p, t, ip);
5677         src = instr_regarray_src_nbo(t, ip);
5678         regarray[idx] += src;
5679
5680         /* Thread. */
5681         thread_ip_inc(p);
5682 }
5683
5684 static inline void
5685 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
5686 {
5687         struct thread *t = &p->threads[p->thread_id];
5688         struct instruction *ip = t->ip;
5689         uint64_t *regarray, idx, src;
5690
5691         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
5692
5693         /* Structs. */
5694         regarray = instr_regarray_regarray(p, ip);
5695         idx = instr_regarray_idx_nbo(p, t, ip);
5696         src = instr_regarray_src_hbo(t, ip);
5697         regarray[idx] += src;
5698
5699         /* Thread. */
5700         thread_ip_inc(p);
5701 }
5702
5703 static inline void
5704 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
5705 {
5706         struct thread *t = &p->threads[p->thread_id];
5707         struct instruction *ip = t->ip;
5708         uint64_t *regarray, idx, src;
5709
5710         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
5711
5712         /* Structs. */
5713         regarray = instr_regarray_regarray(p, ip);
5714         idx = instr_regarray_idx_hbo(p, t, ip);
5715         src = instr_regarray_src_nbo(t, ip);
5716         regarray[idx] += src;
5717
5718         /* Thread. */
5719         thread_ip_inc(p);
5720 }
5721
5722 static inline void
5723 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
5724 {
5725         struct thread *t = &p->threads[p->thread_id];
5726         struct instruction *ip = t->ip;
5727         uint64_t *regarray, idx, src;
5728
5729         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
5730
5731         /* Structs. */
5732         regarray = instr_regarray_regarray(p, ip);
5733         idx = instr_regarray_idx_hbo(p, t, ip);
5734         src = instr_regarray_src_hbo(t, ip);
5735         regarray[idx] += src;
5736
5737         /* Thread. */
5738         thread_ip_inc(p);
5739 }
5740
5741 static inline void
5742 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
5743 {
5744         struct thread *t = &p->threads[p->thread_id];
5745         struct instruction *ip = t->ip;
5746         uint64_t *regarray, idx, src;
5747
5748         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
5749
5750         /* Structs. */
5751         regarray = instr_regarray_regarray(p, ip);
5752         idx = instr_regarray_idx_nbo(p, t, ip);
5753         src = ip->regarray.dstsrc_val;
5754         regarray[idx] += src;
5755
5756         /* Thread. */
5757         thread_ip_inc(p);
5758 }
5759
5760 static inline void
5761 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
5762 {
5763         struct thread *t = &p->threads[p->thread_id];
5764         struct instruction *ip = t->ip;
5765         uint64_t *regarray, idx, src;
5766
5767         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
5768
5769         /* Structs. */
5770         regarray = instr_regarray_regarray(p, ip);
5771         idx = instr_regarray_idx_hbo(p, t, ip);
5772         src = ip->regarray.dstsrc_val;
5773         regarray[idx] += src;
5774
5775         /* Thread. */
5776         thread_ip_inc(p);
5777 }
5778
5779 static inline void
5780 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
5781 {
5782         struct thread *t = &p->threads[p->thread_id];
5783         struct instruction *ip = t->ip;
5784         uint64_t *regarray, idx, src;
5785
5786         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
5787
5788         /* Structs. */
5789         regarray = instr_regarray_regarray(p, ip);
5790         idx = instr_regarray_idx_imm(p, ip);
5791         src = instr_regarray_src_nbo(t, ip);
5792         regarray[idx] += src;
5793
5794         /* Thread. */
5795         thread_ip_inc(p);
5796 }
5797
5798 static inline void
5799 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
5800 {
5801         struct thread *t = &p->threads[p->thread_id];
5802         struct instruction *ip = t->ip;
5803         uint64_t *regarray, idx, src;
5804
5805         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
5806
5807         /* Structs. */
5808         regarray = instr_regarray_regarray(p, ip);
5809         idx = instr_regarray_idx_imm(p, ip);
5810         src = instr_regarray_src_hbo(t, ip);
5811         regarray[idx] += src;
5812
5813         /* Thread. */
5814         thread_ip_inc(p);
5815 }
5816
5817 static inline void
5818 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
5819 {
5820         struct thread *t = &p->threads[p->thread_id];
5821         struct instruction *ip = t->ip;
5822         uint64_t *regarray, idx, src;
5823
5824         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
5825
5826         /* Structs. */
5827         regarray = instr_regarray_regarray(p, ip);
5828         idx = instr_regarray_idx_imm(p, ip);
5829         src = ip->regarray.dstsrc_val;
5830         regarray[idx] += src;
5831
5832         /* Thread. */
5833         thread_ip_inc(p);
5834 }
5835
5836 /*
5837  * metarray.
5838  */
5839 static struct metarray *
5840 metarray_find(struct rte_swx_pipeline *p, const char *name);
5841
5842 static int
5843 instr_metprefetch_translate(struct rte_swx_pipeline *p,
5844                             struct action *action,
5845                             char **tokens,
5846                             int n_tokens,
5847                             struct instruction *instr,
5848                             struct instruction_data *data __rte_unused)
5849 {
5850         char *metarray = tokens[1], *idx = tokens[2];
5851         struct metarray *m;
5852         struct field *fidx;
5853         uint32_t idx_struct_id, idx_val;
5854
5855         CHECK(n_tokens == 3, EINVAL);
5856
5857         m = metarray_find(p, metarray);
5858         CHECK(m, EINVAL);
5859
5860         /* METPREFETCH_H, METPREFETCH_M. */
5861         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5862         if (fidx) {
5863                 instr->type = INSTR_METPREFETCH_M;
5864                 if (idx[0] == 'h')
5865                         instr->type = INSTR_METPREFETCH_H;
5866
5867                 instr->meter.metarray_id = m->id;
5868                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5869                 instr->meter.idx.n_bits = fidx->n_bits;
5870                 instr->meter.idx.offset = fidx->offset / 8;
5871                 return 0;
5872         }
5873
5874         /* METPREFETCH_I. */
5875         idx_val = strtoul(idx, &idx, 0);
5876         CHECK(!idx[0], EINVAL);
5877
5878         instr->type = INSTR_METPREFETCH_I;
5879         instr->meter.metarray_id = m->id;
5880         instr->meter.idx_val = idx_val;
5881         return 0;
5882 }
5883
5884 static int
5885 instr_meter_translate(struct rte_swx_pipeline *p,
5886                       struct action *action,
5887                       char **tokens,
5888                       int n_tokens,
5889                       struct instruction *instr,
5890                       struct instruction_data *data __rte_unused)
5891 {
5892         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
5893         char *color_in = tokens[4], *color_out = tokens[5];
5894         struct metarray *m;
5895         struct field *fidx, *flength, *fcin, *fcout;
5896         uint32_t idx_struct_id, length_struct_id;
5897         uint32_t color_in_struct_id, color_out_struct_id;
5898
5899         CHECK(n_tokens == 6, EINVAL);
5900
5901         m = metarray_find(p, metarray);
5902         CHECK(m, EINVAL);
5903
5904         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5905
5906         flength = struct_field_parse(p, action, length, &length_struct_id);
5907         CHECK(flength, EINVAL);
5908
5909         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
5910
5911         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
5912         CHECK(fcout, EINVAL);
5913
5914         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
5915         if (fidx && fcin) {
5916                 instr->type = INSTR_METER_MMM;
5917                 if (idx[0] == 'h' && length[0] == 'h')
5918                         instr->type = INSTR_METER_HHM;
5919                 if (idx[0] == 'h' && length[0] != 'h')
5920                         instr->type = INSTR_METER_HMM;
5921                 if (idx[0] != 'h' && length[0] == 'h')
5922                         instr->type = INSTR_METER_MHM;
5923
5924                 instr->meter.metarray_id = m->id;
5925
5926                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5927                 instr->meter.idx.n_bits = fidx->n_bits;
5928                 instr->meter.idx.offset = fidx->offset / 8;
5929
5930                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5931                 instr->meter.length.n_bits = flength->n_bits;
5932                 instr->meter.length.offset = flength->offset / 8;
5933
5934                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5935                 instr->meter.color_in.n_bits = fcin->n_bits;
5936                 instr->meter.color_in.offset = fcin->offset / 8;
5937
5938                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5939                 instr->meter.color_out.n_bits = fcout->n_bits;
5940                 instr->meter.color_out.offset = fcout->offset / 8;
5941
5942                 return 0;
5943         }
5944
5945         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
5946         if (fidx && !fcin) {
5947                 uint32_t color_in_val = strtoul(color_in, &color_in, 0);
5948                 CHECK(!color_in[0], EINVAL);
5949
5950                 instr->type = INSTR_METER_MMI;
5951                 if (idx[0] == 'h' && length[0] == 'h')
5952                         instr->type = INSTR_METER_HHI;
5953                 if (idx[0] == 'h' && length[0] != 'h')
5954                         instr->type = INSTR_METER_HMI;
5955                 if (idx[0] != 'h' && length[0] == 'h')
5956                         instr->type = INSTR_METER_MHI;
5957
5958                 instr->meter.metarray_id = m->id;
5959
5960                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5961                 instr->meter.idx.n_bits = fidx->n_bits;
5962                 instr->meter.idx.offset = fidx->offset / 8;
5963
5964                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5965                 instr->meter.length.n_bits = flength->n_bits;
5966                 instr->meter.length.offset = flength->offset / 8;
5967
5968                 instr->meter.color_in_val = color_in_val;
5969
5970                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5971                 instr->meter.color_out.n_bits = fcout->n_bits;
5972                 instr->meter.color_out.offset = fcout->offset / 8;
5973
5974                 return 0;
5975         }
5976
5977         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5978         if (!fidx && fcin) {
5979                 uint32_t idx_val;
5980
5981                 idx_val = strtoul(idx, &idx, 0);
5982                 CHECK(!idx[0], EINVAL);
5983
5984                 instr->type = INSTR_METER_IMM;
5985                 if (length[0] == 'h')
5986                         instr->type = INSTR_METER_IHM;
5987
5988                 instr->meter.metarray_id = m->id;
5989
5990                 instr->meter.idx_val = idx_val;
5991
5992                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5993                 instr->meter.length.n_bits = flength->n_bits;
5994                 instr->meter.length.offset = flength->offset / 8;
5995
5996                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5997                 instr->meter.color_in.n_bits = fcin->n_bits;
5998                 instr->meter.color_in.offset = fcin->offset / 8;
5999
6000                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6001                 instr->meter.color_out.n_bits = fcout->n_bits;
6002                 instr->meter.color_out.offset = fcout->offset / 8;
6003
6004                 return 0;
6005         }
6006
6007         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
6008         if (!fidx && !fcin) {
6009                 uint32_t idx_val, color_in_val;
6010
6011                 idx_val = strtoul(idx, &idx, 0);
6012                 CHECK(!idx[0], EINVAL);
6013
6014                 color_in_val = strtoul(color_in, &color_in, 0);
6015                 CHECK(!color_in[0], EINVAL);
6016
6017                 instr->type = INSTR_METER_IMI;
6018                 if (length[0] == 'h')
6019                         instr->type = INSTR_METER_IHI;
6020
6021                 instr->meter.metarray_id = m->id;
6022
6023                 instr->meter.idx_val = idx_val;
6024
6025                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6026                 instr->meter.length.n_bits = flength->n_bits;
6027                 instr->meter.length.offset = flength->offset / 8;
6028
6029                 instr->meter.color_in_val = color_in_val;
6030
6031                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6032                 instr->meter.color_out.n_bits = fcout->n_bits;
6033                 instr->meter.color_out.offset = fcout->offset / 8;
6034
6035                 return 0;
6036         }
6037
6038         CHECK(0, EINVAL);
6039 }
6040
6041 static inline struct meter *
6042 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6043 {
6044         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6045
6046         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6047         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6048         uint64_t idx64 = *idx64_ptr;
6049         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
6050         uint64_t idx = idx64 & idx64_mask & r->size_mask;
6051
6052         return &r->metarray[idx];
6053 }
6054
6055 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6056
6057 static inline struct meter *
6058 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6059 {
6060         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6061
6062         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6063         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6064         uint64_t idx64 = *idx64_ptr;
6065         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
6066
6067         return &r->metarray[idx];
6068 }
6069
6070 #else
6071
6072 #define instr_meter_idx_nbo instr_meter_idx_hbo
6073
6074 #endif
6075
6076 static inline struct meter *
6077 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
6078 {
6079         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6080
6081         uint64_t idx =  ip->meter.idx_val & r->size_mask;
6082
6083         return &r->metarray[idx];
6084 }
6085
6086 static inline uint32_t
6087 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
6088 {
6089         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6090         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6091         uint64_t src64 = *src64_ptr;
6092         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
6093         uint64_t src = src64 & src64_mask;
6094
6095         return (uint32_t)src;
6096 }
6097
6098 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6099
6100 static inline uint32_t
6101 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
6102 {
6103         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6104         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6105         uint64_t src64 = *src64_ptr;
6106         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
6107
6108         return (uint32_t)src;
6109 }
6110
6111 #else
6112
6113 #define instr_meter_length_nbo instr_meter_length_hbo
6114
6115 #endif
6116
6117 static inline enum rte_color
6118 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
6119 {
6120         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
6121         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
6122         uint64_t src64 = *src64_ptr;
6123         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
6124         uint64_t src = src64 & src64_mask;
6125
6126         return (enum rte_color)src;
6127 }
6128
6129 static inline void
6130 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
6131 {
6132         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
6133         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
6134         uint64_t dst64 = *dst64_ptr;
6135         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
6136
6137         uint64_t src = (uint64_t)color_out;
6138
6139         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6140 }
6141
6142 static inline void
6143 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
6144 {
6145         struct thread *t = &p->threads[p->thread_id];
6146         struct instruction *ip = t->ip;
6147         struct meter *m;
6148
6149         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
6150
6151         /* Structs. */
6152         m = instr_meter_idx_nbo(p, t, ip);
6153         rte_prefetch0(m);
6154
6155         /* Thread. */
6156         thread_ip_inc(p);
6157 }
6158
6159 static inline void
6160 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
6161 {
6162         struct thread *t = &p->threads[p->thread_id];
6163         struct instruction *ip = t->ip;
6164         struct meter *m;
6165
6166         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
6167
6168         /* Structs. */
6169         m = instr_meter_idx_hbo(p, t, ip);
6170         rte_prefetch0(m);
6171
6172         /* Thread. */
6173         thread_ip_inc(p);
6174 }
6175
6176 static inline void
6177 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
6178 {
6179         struct thread *t = &p->threads[p->thread_id];
6180         struct instruction *ip = t->ip;
6181         struct meter *m;
6182
6183         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
6184
6185         /* Structs. */
6186         m = instr_meter_idx_imm(p, ip);
6187         rte_prefetch0(m);
6188
6189         /* Thread. */
6190         thread_ip_inc(p);
6191 }
6192
6193 static inline void
6194 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
6195 {
6196         struct thread *t = &p->threads[p->thread_id];
6197         struct instruction *ip = t->ip;
6198         struct meter *m;
6199         uint64_t time, n_pkts, n_bytes;
6200         uint32_t length;
6201         enum rte_color color_in, color_out;
6202
6203         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
6204
6205         /* Structs. */
6206         m = instr_meter_idx_nbo(p, t, ip);
6207         rte_prefetch0(m->n_pkts);
6208         time = rte_get_tsc_cycles();
6209         length = instr_meter_length_nbo(t, ip);
6210         color_in = instr_meter_color_in_hbo(t, ip);
6211
6212         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6213                 &m->profile->profile,
6214                 time,
6215                 length,
6216                 color_in);
6217
6218         color_out &= m->color_mask;
6219
6220         n_pkts = m->n_pkts[color_out];
6221         n_bytes = m->n_bytes[color_out];
6222
6223         instr_meter_color_out_hbo_set(t, ip, color_out);
6224
6225         m->n_pkts[color_out] = n_pkts + 1;
6226         m->n_bytes[color_out] = n_bytes + length;
6227
6228         /* Thread. */
6229         thread_ip_inc(p);
6230 }
6231
6232 static inline void
6233 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
6234 {
6235         struct thread *t = &p->threads[p->thread_id];
6236         struct instruction *ip = t->ip;
6237         struct meter *m;
6238         uint64_t time, n_pkts, n_bytes;
6239         uint32_t length;
6240         enum rte_color color_in, color_out;
6241
6242         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
6243
6244         /* Structs. */
6245         m = instr_meter_idx_nbo(p, t, ip);
6246         rte_prefetch0(m->n_pkts);
6247         time = rte_get_tsc_cycles();
6248         length = instr_meter_length_nbo(t, ip);
6249         color_in = (enum rte_color)ip->meter.color_in_val;
6250
6251         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6252                 &m->profile->profile,
6253                 time,
6254                 length,
6255                 color_in);
6256
6257         color_out &= m->color_mask;
6258
6259         n_pkts = m->n_pkts[color_out];
6260         n_bytes = m->n_bytes[color_out];
6261
6262         instr_meter_color_out_hbo_set(t, ip, color_out);
6263
6264         m->n_pkts[color_out] = n_pkts + 1;
6265         m->n_bytes[color_out] = n_bytes + length;
6266
6267         /* Thread. */
6268         thread_ip_inc(p);
6269 }
6270
6271 static inline void
6272 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
6273 {
6274         struct thread *t = &p->threads[p->thread_id];
6275         struct instruction *ip = t->ip;
6276         struct meter *m;
6277         uint64_t time, n_pkts, n_bytes;
6278         uint32_t length;
6279         enum rte_color color_in, color_out;
6280
6281         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
6282
6283         /* Structs. */
6284         m = instr_meter_idx_nbo(p, t, ip);
6285         rte_prefetch0(m->n_pkts);
6286         time = rte_get_tsc_cycles();
6287         length = instr_meter_length_hbo(t, ip);
6288         color_in = instr_meter_color_in_hbo(t, ip);
6289
6290         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6291                 &m->profile->profile,
6292                 time,
6293                 length,
6294                 color_in);
6295
6296         color_out &= m->color_mask;
6297
6298         n_pkts = m->n_pkts[color_out];
6299         n_bytes = m->n_bytes[color_out];
6300
6301         instr_meter_color_out_hbo_set(t, ip, color_out);
6302
6303         m->n_pkts[color_out] = n_pkts + 1;
6304         m->n_bytes[color_out] = n_bytes + length;
6305
6306         /* Thread. */
6307         thread_ip_inc(p);
6308 }
6309 static inline void
6310 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
6311 {
6312         struct thread *t = &p->threads[p->thread_id];
6313         struct instruction *ip = t->ip;
6314         struct meter *m;
6315         uint64_t time, n_pkts, n_bytes;
6316         uint32_t length;
6317         enum rte_color color_in, color_out;
6318
6319         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
6320
6321         /* Structs. */
6322         m = instr_meter_idx_nbo(p, t, ip);
6323         rte_prefetch0(m->n_pkts);
6324         time = rte_get_tsc_cycles();
6325         length = instr_meter_length_hbo(t, ip);
6326         color_in = (enum rte_color)ip->meter.color_in_val;
6327
6328         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6329                 &m->profile->profile,
6330                 time,
6331                 length,
6332                 color_in);
6333
6334         color_out &= m->color_mask;
6335
6336         n_pkts = m->n_pkts[color_out];
6337         n_bytes = m->n_bytes[color_out];
6338
6339         instr_meter_color_out_hbo_set(t, ip, color_out);
6340
6341         m->n_pkts[color_out] = n_pkts + 1;
6342         m->n_bytes[color_out] = n_bytes + length;
6343
6344         /* Thread. */
6345         thread_ip_inc(p);
6346 }
6347
6348 static inline void
6349 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
6350 {
6351         struct thread *t = &p->threads[p->thread_id];
6352         struct instruction *ip = t->ip;
6353         struct meter *m;
6354         uint64_t time, n_pkts, n_bytes;
6355         uint32_t length;
6356         enum rte_color color_in, color_out;
6357
6358         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
6359
6360         /* Structs. */
6361         m = instr_meter_idx_hbo(p, t, ip);
6362         rte_prefetch0(m->n_pkts);
6363         time = rte_get_tsc_cycles();
6364         length = instr_meter_length_nbo(t, ip);
6365         color_in = instr_meter_color_in_hbo(t, ip);
6366
6367         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6368                 &m->profile->profile,
6369                 time,
6370                 length,
6371                 color_in);
6372
6373         color_out &= m->color_mask;
6374
6375         n_pkts = m->n_pkts[color_out];
6376         n_bytes = m->n_bytes[color_out];
6377
6378         instr_meter_color_out_hbo_set(t, ip, color_out);
6379
6380         m->n_pkts[color_out] = n_pkts + 1;
6381         m->n_bytes[color_out] = n_bytes + length;
6382
6383         /* Thread. */
6384         thread_ip_inc(p);
6385 }
6386
6387 static inline void
6388 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
6389 {
6390         struct thread *t = &p->threads[p->thread_id];
6391         struct instruction *ip = t->ip;
6392         struct meter *m;
6393         uint64_t time, n_pkts, n_bytes;
6394         uint32_t length;
6395         enum rte_color color_in, color_out;
6396
6397         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
6398
6399         /* Structs. */
6400         m = instr_meter_idx_hbo(p, t, ip);
6401         rte_prefetch0(m->n_pkts);
6402         time = rte_get_tsc_cycles();
6403         length = instr_meter_length_nbo(t, ip);
6404         color_in = (enum rte_color)ip->meter.color_in_val;
6405
6406         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6407                 &m->profile->profile,
6408                 time,
6409                 length,
6410                 color_in);
6411
6412         color_out &= m->color_mask;
6413
6414         n_pkts = m->n_pkts[color_out];
6415         n_bytes = m->n_bytes[color_out];
6416
6417         instr_meter_color_out_hbo_set(t, ip, color_out);
6418
6419         m->n_pkts[color_out] = n_pkts + 1;
6420         m->n_bytes[color_out] = n_bytes + length;
6421
6422         /* Thread. */
6423         thread_ip_inc(p);
6424 }
6425
6426 static inline void
6427 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
6428 {
6429         struct thread *t = &p->threads[p->thread_id];
6430         struct instruction *ip = t->ip;
6431         struct meter *m;
6432         uint64_t time, n_pkts, n_bytes;
6433         uint32_t length;
6434         enum rte_color color_in, color_out;
6435
6436         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
6437
6438         /* Structs. */
6439         m = instr_meter_idx_hbo(p, t, ip);
6440         rte_prefetch0(m->n_pkts);
6441         time = rte_get_tsc_cycles();
6442         length = instr_meter_length_hbo(t, ip);
6443         color_in = instr_meter_color_in_hbo(t, ip);
6444
6445         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6446                 &m->profile->profile,
6447                 time,
6448                 length,
6449                 color_in);
6450
6451         color_out &= m->color_mask;
6452
6453         n_pkts = m->n_pkts[color_out];
6454         n_bytes = m->n_bytes[color_out];
6455
6456         instr_meter_color_out_hbo_set(t, ip, color_out);
6457
6458         m->n_pkts[color_out] = n_pkts + 1;
6459         m->n_bytes[color_out] = n_bytes + length;
6460
6461         /* Thread. */
6462         thread_ip_inc(p);
6463 }
6464
6465 static inline void
6466 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
6467 {
6468         struct thread *t = &p->threads[p->thread_id];
6469         struct instruction *ip = t->ip;
6470         struct meter *m;
6471         uint64_t time, n_pkts, n_bytes;
6472         uint32_t length;
6473         enum rte_color color_in, color_out;
6474
6475         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
6476
6477         /* Structs. */
6478         m = instr_meter_idx_hbo(p, t, ip);
6479         rte_prefetch0(m->n_pkts);
6480         time = rte_get_tsc_cycles();
6481         length = instr_meter_length_hbo(t, ip);
6482         color_in = (enum rte_color)ip->meter.color_in_val;
6483
6484         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6485                 &m->profile->profile,
6486                 time,
6487                 length,
6488                 color_in);
6489
6490         color_out &= m->color_mask;
6491
6492         n_pkts = m->n_pkts[color_out];
6493         n_bytes = m->n_bytes[color_out];
6494
6495         instr_meter_color_out_hbo_set(t, ip, color_out);
6496
6497         m->n_pkts[color_out] = n_pkts + 1;
6498         m->n_bytes[color_out] = n_bytes + length;
6499
6500         /* Thread. */
6501         thread_ip_inc(p);
6502 }
6503
6504 static inline void
6505 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
6506 {
6507         struct thread *t = &p->threads[p->thread_id];
6508         struct instruction *ip = t->ip;
6509         struct meter *m;
6510         uint64_t time, n_pkts, n_bytes;
6511         uint32_t length;
6512         enum rte_color color_in, color_out;
6513
6514         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
6515
6516         /* Structs. */
6517         m = instr_meter_idx_imm(p, ip);
6518         rte_prefetch0(m->n_pkts);
6519         time = rte_get_tsc_cycles();
6520         length = instr_meter_length_nbo(t, ip);
6521         color_in = instr_meter_color_in_hbo(t, ip);
6522
6523         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6524                 &m->profile->profile,
6525                 time,
6526                 length,
6527                 color_in);
6528
6529         color_out &= m->color_mask;
6530
6531         n_pkts = m->n_pkts[color_out];
6532         n_bytes = m->n_bytes[color_out];
6533
6534         instr_meter_color_out_hbo_set(t, ip, color_out);
6535
6536         m->n_pkts[color_out] = n_pkts + 1;
6537         m->n_bytes[color_out] = n_bytes + length;
6538
6539         /* Thread. */
6540         thread_ip_inc(p);
6541 }
6542
6543 static inline void
6544 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
6545 {
6546         struct thread *t = &p->threads[p->thread_id];
6547         struct instruction *ip = t->ip;
6548         struct meter *m;
6549         uint64_t time, n_pkts, n_bytes;
6550         uint32_t length;
6551         enum rte_color color_in, color_out;
6552
6553         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
6554
6555         /* Structs. */
6556         m = instr_meter_idx_imm(p, ip);
6557         rte_prefetch0(m->n_pkts);
6558         time = rte_get_tsc_cycles();
6559         length = instr_meter_length_nbo(t, ip);
6560         color_in = (enum rte_color)ip->meter.color_in_val;
6561
6562         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6563                 &m->profile->profile,
6564                 time,
6565                 length,
6566                 color_in);
6567
6568         color_out &= m->color_mask;
6569
6570         n_pkts = m->n_pkts[color_out];
6571         n_bytes = m->n_bytes[color_out];
6572
6573         instr_meter_color_out_hbo_set(t, ip, color_out);
6574
6575         m->n_pkts[color_out] = n_pkts + 1;
6576         m->n_bytes[color_out] = n_bytes + length;
6577
6578         /* Thread. */
6579         thread_ip_inc(p);
6580 }
6581
6582 static inline void
6583 instr_meter_imm_exec(struct rte_swx_pipeline *p)
6584 {
6585         struct thread *t = &p->threads[p->thread_id];
6586         struct instruction *ip = t->ip;
6587         struct meter *m;
6588         uint64_t time, n_pkts, n_bytes;
6589         uint32_t length;
6590         enum rte_color color_in, color_out;
6591
6592         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
6593
6594         /* Structs. */
6595         m = instr_meter_idx_imm(p, ip);
6596         rte_prefetch0(m->n_pkts);
6597         time = rte_get_tsc_cycles();
6598         length = instr_meter_length_hbo(t, ip);
6599         color_in = instr_meter_color_in_hbo(t, ip);
6600
6601         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6602                 &m->profile->profile,
6603                 time,
6604                 length,
6605                 color_in);
6606
6607         color_out &= m->color_mask;
6608
6609         n_pkts = m->n_pkts[color_out];
6610         n_bytes = m->n_bytes[color_out];
6611
6612         instr_meter_color_out_hbo_set(t, ip, color_out);
6613
6614         m->n_pkts[color_out] = n_pkts + 1;
6615         m->n_bytes[color_out] = n_bytes + length;
6616
6617         /* Thread. */
6618         thread_ip_inc(p);
6619 }
6620 static inline void
6621 instr_meter_imi_exec(struct rte_swx_pipeline *p)
6622 {
6623         struct thread *t = &p->threads[p->thread_id];
6624         struct instruction *ip = t->ip;
6625         struct meter *m;
6626         uint64_t time, n_pkts, n_bytes;
6627         uint32_t length;
6628         enum rte_color color_in, color_out;
6629
6630         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
6631
6632         /* Structs. */
6633         m = instr_meter_idx_imm(p, ip);
6634         rte_prefetch0(m->n_pkts);
6635         time = rte_get_tsc_cycles();
6636         length = instr_meter_length_hbo(t, ip);
6637         color_in = (enum rte_color)ip->meter.color_in_val;
6638
6639         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6640                 &m->profile->profile,
6641                 time,
6642                 length,
6643                 color_in);
6644
6645         color_out &= m->color_mask;
6646
6647         n_pkts = m->n_pkts[color_out];
6648         n_bytes = m->n_bytes[color_out];
6649
6650         instr_meter_color_out_hbo_set(t, ip, color_out);
6651
6652         m->n_pkts[color_out] = n_pkts + 1;
6653         m->n_bytes[color_out] = n_bytes + length;
6654
6655         /* Thread. */
6656         thread_ip_inc(p);
6657 }
6658
6659 /*
6660  * jmp.
6661  */
6662 static struct action *
6663 action_find(struct rte_swx_pipeline *p, const char *name);
6664
6665 static int
6666 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
6667                     struct action *action __rte_unused,
6668                     char **tokens,
6669                     int n_tokens,
6670                     struct instruction *instr,
6671                     struct instruction_data *data)
6672 {
6673         CHECK(n_tokens == 2, EINVAL);
6674
6675         strcpy(data->jmp_label, tokens[1]);
6676
6677         instr->type = INSTR_JMP;
6678         instr->jmp.ip = NULL; /* Resolved later. */
6679         return 0;
6680 }
6681
6682 static int
6683 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
6684                           struct action *action __rte_unused,
6685                           char **tokens,
6686                           int n_tokens,
6687                           struct instruction *instr,
6688                           struct instruction_data *data)
6689 {
6690         struct header *h;
6691
6692         CHECK(n_tokens == 3, EINVAL);
6693
6694         strcpy(data->jmp_label, tokens[1]);
6695
6696         h = header_parse(p, tokens[2]);
6697         CHECK(h, EINVAL);
6698
6699         instr->type = INSTR_JMP_VALID;
6700         instr->jmp.ip = NULL; /* Resolved later. */
6701         instr->jmp.header_id = h->id;
6702         return 0;
6703 }
6704
6705 static int
6706 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
6707                             struct action *action __rte_unused,
6708                             char **tokens,
6709                             int n_tokens,
6710                             struct instruction *instr,
6711                             struct instruction_data *data)
6712 {
6713         struct header *h;
6714
6715         CHECK(n_tokens == 3, EINVAL);
6716
6717         strcpy(data->jmp_label, tokens[1]);
6718
6719         h = header_parse(p, tokens[2]);
6720         CHECK(h, EINVAL);
6721
6722         instr->type = INSTR_JMP_INVALID;
6723         instr->jmp.ip = NULL; /* Resolved later. */
6724         instr->jmp.header_id = h->id;
6725         return 0;
6726 }
6727
6728 static int
6729 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
6730                         struct action *action,
6731                         char **tokens,
6732                         int n_tokens,
6733                         struct instruction *instr,
6734                         struct instruction_data *data)
6735 {
6736         CHECK(!action, EINVAL);
6737         CHECK(n_tokens == 2, EINVAL);
6738
6739         strcpy(data->jmp_label, tokens[1]);
6740
6741         instr->type = INSTR_JMP_HIT;
6742         instr->jmp.ip = NULL; /* Resolved later. */
6743         return 0;
6744 }
6745
6746 static int
6747 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
6748                          struct action *action,
6749                          char **tokens,
6750                          int n_tokens,
6751                          struct instruction *instr,
6752                          struct instruction_data *data)
6753 {
6754         CHECK(!action, EINVAL);
6755         CHECK(n_tokens == 2, EINVAL);
6756
6757         strcpy(data->jmp_label, tokens[1]);
6758
6759         instr->type = INSTR_JMP_MISS;
6760         instr->jmp.ip = NULL; /* Resolved later. */
6761         return 0;
6762 }
6763
6764 static int
6765 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
6766                                struct action *action,
6767                                char **tokens,
6768                                int n_tokens,
6769                                struct instruction *instr,
6770                                struct instruction_data *data)
6771 {
6772         struct action *a;
6773
6774         CHECK(!action, EINVAL);
6775         CHECK(n_tokens == 3, EINVAL);
6776
6777         strcpy(data->jmp_label, tokens[1]);
6778
6779         a = action_find(p, tokens[2]);
6780         CHECK(a, EINVAL);
6781
6782         instr->type = INSTR_JMP_ACTION_HIT;
6783         instr->jmp.ip = NULL; /* Resolved later. */
6784         instr->jmp.action_id = a->id;
6785         return 0;
6786 }
6787
6788 static int
6789 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
6790                                 struct action *action,
6791                                 char **tokens,
6792                                 int n_tokens,
6793                                 struct instruction *instr,
6794                                 struct instruction_data *data)
6795 {
6796         struct action *a;
6797
6798         CHECK(!action, EINVAL);
6799         CHECK(n_tokens == 3, EINVAL);
6800
6801         strcpy(data->jmp_label, tokens[1]);
6802
6803         a = action_find(p, tokens[2]);
6804         CHECK(a, EINVAL);
6805
6806         instr->type = INSTR_JMP_ACTION_MISS;
6807         instr->jmp.ip = NULL; /* Resolved later. */
6808         instr->jmp.action_id = a->id;
6809         return 0;
6810 }
6811
6812 static int
6813 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
6814                        struct action *action,
6815                        char **tokens,
6816                        int n_tokens,
6817                        struct instruction *instr,
6818                        struct instruction_data *data)
6819 {
6820         char *a = tokens[2], *b = tokens[3];
6821         struct field *fa, *fb;
6822         uint64_t b_val;
6823         uint32_t a_struct_id, b_struct_id;
6824
6825         CHECK(n_tokens == 4, EINVAL);
6826
6827         strcpy(data->jmp_label, tokens[1]);
6828
6829         fa = struct_field_parse(p, action, a, &a_struct_id);
6830         CHECK(fa, EINVAL);
6831
6832         /* JMP_EQ or JMP_EQ_S. */
6833         fb = struct_field_parse(p, action, b, &b_struct_id);
6834         if (fb) {
6835                 instr->type = INSTR_JMP_EQ;
6836                 if ((a[0] == 'h' && b[0] != 'h') ||
6837                     (a[0] != 'h' && b[0] == 'h'))
6838                         instr->type = INSTR_JMP_EQ_S;
6839                 instr->jmp.ip = NULL; /* Resolved later. */
6840
6841                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6842                 instr->jmp.a.n_bits = fa->n_bits;
6843                 instr->jmp.a.offset = fa->offset / 8;
6844                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6845                 instr->jmp.b.n_bits = fb->n_bits;
6846                 instr->jmp.b.offset = fb->offset / 8;
6847                 return 0;
6848         }
6849
6850         /* JMP_EQ_I. */
6851         b_val = strtoull(b, &b, 0);
6852         CHECK(!b[0], EINVAL);
6853
6854         if (a[0] == 'h')
6855                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6856
6857         instr->type = INSTR_JMP_EQ_I;
6858         instr->jmp.ip = NULL; /* Resolved later. */
6859         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6860         instr->jmp.a.n_bits = fa->n_bits;
6861         instr->jmp.a.offset = fa->offset / 8;
6862         instr->jmp.b_val = b_val;
6863         return 0;
6864 }
6865
6866 static int
6867 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
6868                         struct action *action,
6869                         char **tokens,
6870                         int n_tokens,
6871                         struct instruction *instr,
6872                         struct instruction_data *data)
6873 {
6874         char *a = tokens[2], *b = tokens[3];
6875         struct field *fa, *fb;
6876         uint64_t b_val;
6877         uint32_t a_struct_id, b_struct_id;
6878
6879         CHECK(n_tokens == 4, EINVAL);
6880
6881         strcpy(data->jmp_label, tokens[1]);
6882
6883         fa = struct_field_parse(p, action, a, &a_struct_id);
6884         CHECK(fa, EINVAL);
6885
6886         /* JMP_NEQ or JMP_NEQ_S. */
6887         fb = struct_field_parse(p, action, b, &b_struct_id);
6888         if (fb) {
6889                 instr->type = INSTR_JMP_NEQ;
6890                 if ((a[0] == 'h' && b[0] != 'h') ||
6891                     (a[0] != 'h' && b[0] == 'h'))
6892                         instr->type = INSTR_JMP_NEQ_S;
6893                 instr->jmp.ip = NULL; /* Resolved later. */
6894
6895                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6896                 instr->jmp.a.n_bits = fa->n_bits;
6897                 instr->jmp.a.offset = fa->offset / 8;
6898                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6899                 instr->jmp.b.n_bits = fb->n_bits;
6900                 instr->jmp.b.offset = fb->offset / 8;
6901                 return 0;
6902         }
6903
6904         /* JMP_NEQ_I. */
6905         b_val = strtoull(b, &b, 0);
6906         CHECK(!b[0], EINVAL);
6907
6908         if (a[0] == 'h')
6909                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6910
6911         instr->type = INSTR_JMP_NEQ_I;
6912         instr->jmp.ip = NULL; /* Resolved later. */
6913         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6914         instr->jmp.a.n_bits = fa->n_bits;
6915         instr->jmp.a.offset = fa->offset / 8;
6916         instr->jmp.b_val = b_val;
6917         return 0;
6918 }
6919
6920 static int
6921 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
6922                        struct action *action,
6923                        char **tokens,
6924                        int n_tokens,
6925                        struct instruction *instr,
6926                        struct instruction_data *data)
6927 {
6928         char *a = tokens[2], *b = tokens[3];
6929         struct field *fa, *fb;
6930         uint64_t b_val;
6931         uint32_t a_struct_id, b_struct_id;
6932
6933         CHECK(n_tokens == 4, EINVAL);
6934
6935         strcpy(data->jmp_label, tokens[1]);
6936
6937         fa = struct_field_parse(p, action, a, &a_struct_id);
6938         CHECK(fa, EINVAL);
6939
6940         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
6941         fb = struct_field_parse(p, action, b, &b_struct_id);
6942         if (fb) {
6943                 instr->type = INSTR_JMP_LT;
6944                 if (a[0] == 'h' && b[0] != 'h')
6945                         instr->type = INSTR_JMP_LT_HM;
6946                 if (a[0] != 'h' && b[0] == 'h')
6947                         instr->type = INSTR_JMP_LT_MH;
6948                 if (a[0] == 'h' && b[0] == 'h')
6949                         instr->type = INSTR_JMP_LT_HH;
6950                 instr->jmp.ip = NULL; /* Resolved later. */
6951
6952                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6953                 instr->jmp.a.n_bits = fa->n_bits;
6954                 instr->jmp.a.offset = fa->offset / 8;
6955                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6956                 instr->jmp.b.n_bits = fb->n_bits;
6957                 instr->jmp.b.offset = fb->offset / 8;
6958                 return 0;
6959         }
6960
6961         /* JMP_LT_MI, JMP_LT_HI. */
6962         b_val = strtoull(b, &b, 0);
6963         CHECK(!b[0], EINVAL);
6964
6965         instr->type = INSTR_JMP_LT_MI;
6966         if (a[0] == 'h')
6967                 instr->type = INSTR_JMP_LT_HI;
6968         instr->jmp.ip = NULL; /* Resolved later. */
6969
6970         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6971         instr->jmp.a.n_bits = fa->n_bits;
6972         instr->jmp.a.offset = fa->offset / 8;
6973         instr->jmp.b_val = b_val;
6974         return 0;
6975 }
6976
6977 static int
6978 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
6979                        struct action *action,
6980                        char **tokens,
6981                        int n_tokens,
6982                        struct instruction *instr,
6983                        struct instruction_data *data)
6984 {
6985         char *a = tokens[2], *b = tokens[3];
6986         struct field *fa, *fb;
6987         uint64_t b_val;
6988         uint32_t a_struct_id, b_struct_id;
6989
6990         CHECK(n_tokens == 4, EINVAL);
6991
6992         strcpy(data->jmp_label, tokens[1]);
6993
6994         fa = struct_field_parse(p, action, a, &a_struct_id);
6995         CHECK(fa, EINVAL);
6996
6997         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
6998         fb = struct_field_parse(p, action, b, &b_struct_id);
6999         if (fb) {
7000                 instr->type = INSTR_JMP_GT;
7001                 if (a[0] == 'h' && b[0] != 'h')
7002                         instr->type = INSTR_JMP_GT_HM;
7003                 if (a[0] != 'h' && b[0] == 'h')
7004                         instr->type = INSTR_JMP_GT_MH;
7005                 if (a[0] == 'h' && b[0] == 'h')
7006                         instr->type = INSTR_JMP_GT_HH;
7007                 instr->jmp.ip = NULL; /* Resolved later. */
7008
7009                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7010                 instr->jmp.a.n_bits = fa->n_bits;
7011                 instr->jmp.a.offset = fa->offset / 8;
7012                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7013                 instr->jmp.b.n_bits = fb->n_bits;
7014                 instr->jmp.b.offset = fb->offset / 8;
7015                 return 0;
7016         }
7017
7018         /* JMP_GT_MI, JMP_GT_HI. */
7019         b_val = strtoull(b, &b, 0);
7020         CHECK(!b[0], EINVAL);
7021
7022         instr->type = INSTR_JMP_GT_MI;
7023         if (a[0] == 'h')
7024                 instr->type = INSTR_JMP_GT_HI;
7025         instr->jmp.ip = NULL; /* Resolved later. */
7026
7027         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7028         instr->jmp.a.n_bits = fa->n_bits;
7029         instr->jmp.a.offset = fa->offset / 8;
7030         instr->jmp.b_val = b_val;
7031         return 0;
7032 }
7033
7034 static inline void
7035 instr_jmp_exec(struct rte_swx_pipeline *p)
7036 {
7037         struct thread *t = &p->threads[p->thread_id];
7038         struct instruction *ip = t->ip;
7039
7040         TRACE("[Thread %2u] jmp\n", p->thread_id);
7041
7042         thread_ip_set(t, ip->jmp.ip);
7043 }
7044
7045 static inline void
7046 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
7047 {
7048         struct thread *t = &p->threads[p->thread_id];
7049         struct instruction *ip = t->ip;
7050         uint32_t header_id = ip->jmp.header_id;
7051
7052         TRACE("[Thread %2u] jmpv\n", p->thread_id);
7053
7054         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
7055 }
7056
7057 static inline void
7058 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
7059 {
7060         struct thread *t = &p->threads[p->thread_id];
7061         struct instruction *ip = t->ip;
7062         uint32_t header_id = ip->jmp.header_id;
7063
7064         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
7065
7066         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
7067 }
7068
7069 static inline void
7070 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
7071 {
7072         struct thread *t = &p->threads[p->thread_id];
7073         struct instruction *ip = t->ip;
7074         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
7075
7076         TRACE("[Thread %2u] jmph\n", p->thread_id);
7077
7078         t->ip = ip_next[t->hit];
7079 }
7080
7081 static inline void
7082 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
7083 {
7084         struct thread *t = &p->threads[p->thread_id];
7085         struct instruction *ip = t->ip;
7086         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
7087
7088         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
7089
7090         t->ip = ip_next[t->hit];
7091 }
7092
7093 static inline void
7094 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
7095 {
7096         struct thread *t = &p->threads[p->thread_id];
7097         struct instruction *ip = t->ip;
7098
7099         TRACE("[Thread %2u] jmpa\n", p->thread_id);
7100
7101         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
7102 }
7103
7104 static inline void
7105 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
7106 {
7107         struct thread *t = &p->threads[p->thread_id];
7108         struct instruction *ip = t->ip;
7109
7110         TRACE("[Thread %2u] jmpna\n", p->thread_id);
7111
7112         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
7113 }
7114
7115 static inline void
7116 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
7117 {
7118         struct thread *t = &p->threads[p->thread_id];
7119         struct instruction *ip = t->ip;
7120
7121         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
7122
7123         JMP_CMP(t, ip, ==);
7124 }
7125
7126 static inline void
7127 instr_jmp_eq_s_exec(struct rte_swx_pipeline *p)
7128 {
7129         struct thread *t = &p->threads[p->thread_id];
7130         struct instruction *ip = t->ip;
7131
7132         TRACE("[Thread %2u] jmpeq (s)\n", p->thread_id);
7133
7134         JMP_CMP_S(t, ip, ==);
7135 }
7136
7137 static inline void
7138 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
7139 {
7140         struct thread *t = &p->threads[p->thread_id];
7141         struct instruction *ip = t->ip;
7142
7143         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
7144
7145         JMP_CMP_I(t, ip, ==);
7146 }
7147
7148 static inline void
7149 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
7150 {
7151         struct thread *t = &p->threads[p->thread_id];
7152         struct instruction *ip = t->ip;
7153
7154         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
7155
7156         JMP_CMP(t, ip, !=);
7157 }
7158
7159 static inline void
7160 instr_jmp_neq_s_exec(struct rte_swx_pipeline *p)
7161 {
7162         struct thread *t = &p->threads[p->thread_id];
7163         struct instruction *ip = t->ip;
7164
7165         TRACE("[Thread %2u] jmpneq (s)\n", p->thread_id);
7166
7167         JMP_CMP_S(t, ip, !=);
7168 }
7169
7170 static inline void
7171 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
7172 {
7173         struct thread *t = &p->threads[p->thread_id];
7174         struct instruction *ip = t->ip;
7175
7176         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
7177
7178         JMP_CMP_I(t, ip, !=);
7179 }
7180
7181 static inline void
7182 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
7183 {
7184         struct thread *t = &p->threads[p->thread_id];
7185         struct instruction *ip = t->ip;
7186
7187         TRACE("[Thread %2u] jmplt\n", p->thread_id);
7188
7189         JMP_CMP(t, ip, <);
7190 }
7191
7192 static inline void
7193 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
7194 {
7195         struct thread *t = &p->threads[p->thread_id];
7196         struct instruction *ip = t->ip;
7197
7198         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
7199
7200         JMP_CMP_MH(t, ip, <);
7201 }
7202
7203 static inline void
7204 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
7205 {
7206         struct thread *t = &p->threads[p->thread_id];
7207         struct instruction *ip = t->ip;
7208
7209         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
7210
7211         JMP_CMP_HM(t, ip, <);
7212 }
7213
7214 static inline void
7215 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
7216 {
7217         struct thread *t = &p->threads[p->thread_id];
7218         struct instruction *ip = t->ip;
7219
7220         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
7221
7222         JMP_CMP_HH(t, ip, <);
7223 }
7224
7225 static inline void
7226 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
7227 {
7228         struct thread *t = &p->threads[p->thread_id];
7229         struct instruction *ip = t->ip;
7230
7231         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
7232
7233         JMP_CMP_MI(t, ip, <);
7234 }
7235
7236 static inline void
7237 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
7238 {
7239         struct thread *t = &p->threads[p->thread_id];
7240         struct instruction *ip = t->ip;
7241
7242         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
7243
7244         JMP_CMP_HI(t, ip, <);
7245 }
7246
7247 static inline void
7248 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
7249 {
7250         struct thread *t = &p->threads[p->thread_id];
7251         struct instruction *ip = t->ip;
7252
7253         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
7254
7255         JMP_CMP(t, ip, >);
7256 }
7257
7258 static inline void
7259 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
7260 {
7261         struct thread *t = &p->threads[p->thread_id];
7262         struct instruction *ip = t->ip;
7263
7264         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
7265
7266         JMP_CMP_MH(t, ip, >);
7267 }
7268
7269 static inline void
7270 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
7271 {
7272         struct thread *t = &p->threads[p->thread_id];
7273         struct instruction *ip = t->ip;
7274
7275         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
7276
7277         JMP_CMP_HM(t, ip, >);
7278 }
7279
7280 static inline void
7281 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
7282 {
7283         struct thread *t = &p->threads[p->thread_id];
7284         struct instruction *ip = t->ip;
7285
7286         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
7287
7288         JMP_CMP_HH(t, ip, >);
7289 }
7290
7291 static inline void
7292 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
7293 {
7294         struct thread *t = &p->threads[p->thread_id];
7295         struct instruction *ip = t->ip;
7296
7297         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
7298
7299         JMP_CMP_MI(t, ip, >);
7300 }
7301
7302 static inline void
7303 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
7304 {
7305         struct thread *t = &p->threads[p->thread_id];
7306         struct instruction *ip = t->ip;
7307
7308         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
7309
7310         JMP_CMP_HI(t, ip, >);
7311 }
7312
7313 /*
7314  * return.
7315  */
7316 static int
7317 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
7318                        struct action *action,
7319                        char **tokens __rte_unused,
7320                        int n_tokens,
7321                        struct instruction *instr,
7322                        struct instruction_data *data __rte_unused)
7323 {
7324         CHECK(action, EINVAL);
7325         CHECK(n_tokens == 1, EINVAL);
7326
7327         instr->type = INSTR_RETURN;
7328         return 0;
7329 }
7330
7331 static inline void
7332 instr_return_exec(struct rte_swx_pipeline *p)
7333 {
7334         struct thread *t = &p->threads[p->thread_id];
7335
7336         TRACE("[Thread %2u] return\n", p->thread_id);
7337
7338         t->ip = t->ret;
7339 }
7340
7341 static int
7342 instr_translate(struct rte_swx_pipeline *p,
7343                 struct action *action,
7344                 char *string,
7345                 struct instruction *instr,
7346                 struct instruction_data *data)
7347 {
7348         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
7349         int n_tokens = 0, tpos = 0;
7350
7351         /* Parse the instruction string into tokens. */
7352         for ( ; ; ) {
7353                 char *token;
7354
7355                 token = strtok_r(string, " \t\v", &string);
7356                 if (!token)
7357                         break;
7358
7359                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
7360                 CHECK_NAME(token, EINVAL);
7361
7362                 tokens[n_tokens] = token;
7363                 n_tokens++;
7364         }
7365
7366         CHECK(n_tokens, EINVAL);
7367
7368         /* Handle the optional instruction label. */
7369         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
7370                 strcpy(data->label, tokens[0]);
7371
7372                 tpos += 2;
7373                 CHECK(n_tokens - tpos, EINVAL);
7374         }
7375
7376         /* Identify the instruction type. */
7377         if (!strcmp(tokens[tpos], "rx"))
7378                 return instr_rx_translate(p,
7379                                           action,
7380                                           &tokens[tpos],
7381                                           n_tokens - tpos,
7382                                           instr,
7383                                           data);
7384
7385         if (!strcmp(tokens[tpos], "tx"))
7386                 return instr_tx_translate(p,
7387                                           action,
7388                                           &tokens[tpos],
7389                                           n_tokens - tpos,
7390                                           instr,
7391                                           data);
7392
7393         if (!strcmp(tokens[tpos], "drop"))
7394                 return instr_drop_translate(p,
7395                                             action,
7396                                             &tokens[tpos],
7397                                             n_tokens - tpos,
7398                                             instr,
7399                                             data);
7400
7401         if (!strcmp(tokens[tpos], "extract"))
7402                 return instr_hdr_extract_translate(p,
7403                                                    action,
7404                                                    &tokens[tpos],
7405                                                    n_tokens - tpos,
7406                                                    instr,
7407                                                    data);
7408
7409         if (!strcmp(tokens[tpos], "emit"))
7410                 return instr_hdr_emit_translate(p,
7411                                                 action,
7412                                                 &tokens[tpos],
7413                                                 n_tokens - tpos,
7414                                                 instr,
7415                                                 data);
7416
7417         if (!strcmp(tokens[tpos], "validate"))
7418                 return instr_hdr_validate_translate(p,
7419                                                     action,
7420                                                     &tokens[tpos],
7421                                                     n_tokens - tpos,
7422                                                     instr,
7423                                                     data);
7424
7425         if (!strcmp(tokens[tpos], "invalidate"))
7426                 return instr_hdr_invalidate_translate(p,
7427                                                       action,
7428                                                       &tokens[tpos],
7429                                                       n_tokens - tpos,
7430                                                       instr,
7431                                                       data);
7432
7433         if (!strcmp(tokens[tpos], "mov"))
7434                 return instr_mov_translate(p,
7435                                            action,
7436                                            &tokens[tpos],
7437                                            n_tokens - tpos,
7438                                            instr,
7439                                            data);
7440
7441         if (!strcmp(tokens[tpos], "dma"))
7442                 return instr_dma_translate(p,
7443                                            action,
7444                                            &tokens[tpos],
7445                                            n_tokens - tpos,
7446                                            instr,
7447                                            data);
7448
7449         if (!strcmp(tokens[tpos], "add"))
7450                 return instr_alu_add_translate(p,
7451                                                action,
7452                                                &tokens[tpos],
7453                                                n_tokens - tpos,
7454                                                instr,
7455                                                data);
7456
7457         if (!strcmp(tokens[tpos], "sub"))
7458                 return instr_alu_sub_translate(p,
7459                                                action,
7460                                                &tokens[tpos],
7461                                                n_tokens - tpos,
7462                                                instr,
7463                                                data);
7464
7465         if (!strcmp(tokens[tpos], "ckadd"))
7466                 return instr_alu_ckadd_translate(p,
7467                                                  action,
7468                                                  &tokens[tpos],
7469                                                  n_tokens - tpos,
7470                                                  instr,
7471                                                  data);
7472
7473         if (!strcmp(tokens[tpos], "cksub"))
7474                 return instr_alu_cksub_translate(p,
7475                                                  action,
7476                                                  &tokens[tpos],
7477                                                  n_tokens - tpos,
7478                                                  instr,
7479                                                  data);
7480
7481         if (!strcmp(tokens[tpos], "and"))
7482                 return instr_alu_and_translate(p,
7483                                                action,
7484                                                &tokens[tpos],
7485                                                n_tokens - tpos,
7486                                                instr,
7487                                                data);
7488
7489         if (!strcmp(tokens[tpos], "or"))
7490                 return instr_alu_or_translate(p,
7491                                               action,
7492                                               &tokens[tpos],
7493                                               n_tokens - tpos,
7494                                               instr,
7495                                               data);
7496
7497         if (!strcmp(tokens[tpos], "xor"))
7498                 return instr_alu_xor_translate(p,
7499                                                action,
7500                                                &tokens[tpos],
7501                                                n_tokens - tpos,
7502                                                instr,
7503                                                data);
7504
7505         if (!strcmp(tokens[tpos], "shl"))
7506                 return instr_alu_shl_translate(p,
7507                                                action,
7508                                                &tokens[tpos],
7509                                                n_tokens - tpos,
7510                                                instr,
7511                                                data);
7512
7513         if (!strcmp(tokens[tpos], "shr"))
7514                 return instr_alu_shr_translate(p,
7515                                                action,
7516                                                &tokens[tpos],
7517                                                n_tokens - tpos,
7518                                                instr,
7519                                                data);
7520
7521         if (!strcmp(tokens[tpos], "regprefetch"))
7522                 return instr_regprefetch_translate(p,
7523                                                    action,
7524                                                    &tokens[tpos],
7525                                                    n_tokens - tpos,
7526                                                    instr,
7527                                                    data);
7528
7529         if (!strcmp(tokens[tpos], "regrd"))
7530                 return instr_regrd_translate(p,
7531                                              action,
7532                                              &tokens[tpos],
7533                                              n_tokens - tpos,
7534                                              instr,
7535                                              data);
7536
7537         if (!strcmp(tokens[tpos], "regwr"))
7538                 return instr_regwr_translate(p,
7539                                              action,
7540                                              &tokens[tpos],
7541                                              n_tokens - tpos,
7542                                              instr,
7543                                              data);
7544
7545         if (!strcmp(tokens[tpos], "regadd"))
7546                 return instr_regadd_translate(p,
7547                                               action,
7548                                               &tokens[tpos],
7549                                               n_tokens - tpos,
7550                                               instr,
7551                                               data);
7552
7553         if (!strcmp(tokens[tpos], "metprefetch"))
7554                 return instr_metprefetch_translate(p,
7555                                                    action,
7556                                                    &tokens[tpos],
7557                                                    n_tokens - tpos,
7558                                                    instr,
7559                                                    data);
7560
7561         if (!strcmp(tokens[tpos], "meter"))
7562                 return instr_meter_translate(p,
7563                                              action,
7564                                              &tokens[tpos],
7565                                              n_tokens - tpos,
7566                                              instr,
7567                                              data);
7568
7569         if (!strcmp(tokens[tpos], "table"))
7570                 return instr_table_translate(p,
7571                                              action,
7572                                              &tokens[tpos],
7573                                              n_tokens - tpos,
7574                                              instr,
7575                                              data);
7576
7577         if (!strcmp(tokens[tpos], "extern"))
7578                 return instr_extern_translate(p,
7579                                               action,
7580                                               &tokens[tpos],
7581                                               n_tokens - tpos,
7582                                               instr,
7583                                               data);
7584
7585         if (!strcmp(tokens[tpos], "jmp"))
7586                 return instr_jmp_translate(p,
7587                                            action,
7588                                            &tokens[tpos],
7589                                            n_tokens - tpos,
7590                                            instr,
7591                                            data);
7592
7593         if (!strcmp(tokens[tpos], "jmpv"))
7594                 return instr_jmp_valid_translate(p,
7595                                                  action,
7596                                                  &tokens[tpos],
7597                                                  n_tokens - tpos,
7598                                                  instr,
7599                                                  data);
7600
7601         if (!strcmp(tokens[tpos], "jmpnv"))
7602                 return instr_jmp_invalid_translate(p,
7603                                                    action,
7604                                                    &tokens[tpos],
7605                                                    n_tokens - tpos,
7606                                                    instr,
7607                                                    data);
7608
7609         if (!strcmp(tokens[tpos], "jmph"))
7610                 return instr_jmp_hit_translate(p,
7611                                                action,
7612                                                &tokens[tpos],
7613                                                n_tokens - tpos,
7614                                                instr,
7615                                                data);
7616
7617         if (!strcmp(tokens[tpos], "jmpnh"))
7618                 return instr_jmp_miss_translate(p,
7619                                                 action,
7620                                                 &tokens[tpos],
7621                                                 n_tokens - tpos,
7622                                                 instr,
7623                                                 data);
7624
7625         if (!strcmp(tokens[tpos], "jmpa"))
7626                 return instr_jmp_action_hit_translate(p,
7627                                                       action,
7628                                                       &tokens[tpos],
7629                                                       n_tokens - tpos,
7630                                                       instr,
7631                                                       data);
7632
7633         if (!strcmp(tokens[tpos], "jmpna"))
7634                 return instr_jmp_action_miss_translate(p,
7635                                                        action,
7636                                                        &tokens[tpos],
7637                                                        n_tokens - tpos,
7638                                                        instr,
7639                                                        data);
7640
7641         if (!strcmp(tokens[tpos], "jmpeq"))
7642                 return instr_jmp_eq_translate(p,
7643                                               action,
7644                                               &tokens[tpos],
7645                                               n_tokens - tpos,
7646                                               instr,
7647                                               data);
7648
7649         if (!strcmp(tokens[tpos], "jmpneq"))
7650                 return instr_jmp_neq_translate(p,
7651                                                action,
7652                                                &tokens[tpos],
7653                                                n_tokens - tpos,
7654                                                instr,
7655                                                data);
7656
7657         if (!strcmp(tokens[tpos], "jmplt"))
7658                 return instr_jmp_lt_translate(p,
7659                                               action,
7660                                               &tokens[tpos],
7661                                               n_tokens - tpos,
7662                                               instr,
7663                                               data);
7664
7665         if (!strcmp(tokens[tpos], "jmpgt"))
7666                 return instr_jmp_gt_translate(p,
7667                                               action,
7668                                               &tokens[tpos],
7669                                               n_tokens - tpos,
7670                                               instr,
7671                                               data);
7672
7673         if (!strcmp(tokens[tpos], "return"))
7674                 return instr_return_translate(p,
7675                                               action,
7676                                               &tokens[tpos],
7677                                               n_tokens - tpos,
7678                                               instr,
7679                                               data);
7680
7681         CHECK(0, EINVAL);
7682 }
7683
7684 static struct instruction_data *
7685 label_find(struct instruction_data *data, uint32_t n, const char *label)
7686 {
7687         uint32_t i;
7688
7689         for (i = 0; i < n; i++)
7690                 if (!strcmp(label, data[i].label))
7691                         return &data[i];
7692
7693         return NULL;
7694 }
7695
7696 static uint32_t
7697 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
7698 {
7699         uint32_t count = 0, i;
7700
7701         if (!label[0])
7702                 return 0;
7703
7704         for (i = 0; i < n; i++)
7705                 if (!strcmp(label, data[i].jmp_label))
7706                         count++;
7707
7708         return count;
7709 }
7710
7711 static int
7712 instr_label_check(struct instruction_data *instruction_data,
7713                   uint32_t n_instructions)
7714 {
7715         uint32_t i;
7716
7717         /* Check that all instruction labels are unique. */
7718         for (i = 0; i < n_instructions; i++) {
7719                 struct instruction_data *data = &instruction_data[i];
7720                 char *label = data->label;
7721                 uint32_t j;
7722
7723                 if (!label[0])
7724                         continue;
7725
7726                 for (j = i + 1; j < n_instructions; j++)
7727                         CHECK(strcmp(label, data[j].label), EINVAL);
7728         }
7729
7730         /* Get users for each instruction label. */
7731         for (i = 0; i < n_instructions; i++) {
7732                 struct instruction_data *data = &instruction_data[i];
7733                 char *label = data->label;
7734
7735                 data->n_users = label_is_used(instruction_data,
7736                                               n_instructions,
7737                                               label);
7738         }
7739
7740         return 0;
7741 }
7742
7743 static int
7744 instr_jmp_resolve(struct instruction *instructions,
7745                   struct instruction_data *instruction_data,
7746                   uint32_t n_instructions)
7747 {
7748         uint32_t i;
7749
7750         for (i = 0; i < n_instructions; i++) {
7751                 struct instruction *instr = &instructions[i];
7752                 struct instruction_data *data = &instruction_data[i];
7753                 struct instruction_data *found;
7754
7755                 if (!instruction_is_jmp(instr))
7756                         continue;
7757
7758                 found = label_find(instruction_data,
7759                                    n_instructions,
7760                                    data->jmp_label);
7761                 CHECK(found, EINVAL);
7762
7763                 instr->jmp.ip = &instructions[found - instruction_data];
7764         }
7765
7766         return 0;
7767 }
7768
7769 static int
7770 instr_verify(struct rte_swx_pipeline *p __rte_unused,
7771              struct action *a,
7772              struct instruction *instr,
7773              struct instruction_data *data __rte_unused,
7774              uint32_t n_instructions)
7775 {
7776         if (!a) {
7777                 enum instruction_type type;
7778                 uint32_t i;
7779
7780                 /* Check that the first instruction is rx. */
7781                 CHECK(instr[0].type == INSTR_RX, EINVAL);
7782
7783                 /* Check that there is at least one tx instruction. */
7784                 for (i = 0; i < n_instructions; i++) {
7785                         type = instr[i].type;
7786
7787                         if (instruction_is_tx(type))
7788                                 break;
7789                 }
7790                 CHECK(i < n_instructions, EINVAL);
7791
7792                 /* Check that the last instruction is either tx or unconditional
7793                  * jump.
7794                  */
7795                 type = instr[n_instructions - 1].type;
7796                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
7797         }
7798
7799         if (a) {
7800                 enum instruction_type type;
7801                 uint32_t i;
7802
7803                 /* Check that there is at least one return or tx instruction. */
7804                 for (i = 0; i < n_instructions; i++) {
7805                         type = instr[i].type;
7806
7807                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
7808                                 break;
7809                 }
7810                 CHECK(i < n_instructions, EINVAL);
7811         }
7812
7813         return 0;
7814 }
7815
7816 static int
7817 instr_pattern_extract_many_detect(struct instruction *instr,
7818                                   struct instruction_data *data,
7819                                   uint32_t n_instr,
7820                                   uint32_t *n_pattern_instr)
7821 {
7822         uint32_t i;
7823
7824         for (i = 0; i < n_instr; i++) {
7825                 if (data[i].invalid)
7826                         break;
7827
7828                 if (instr[i].type != INSTR_HDR_EXTRACT)
7829                         break;
7830
7831                 if (i == RTE_DIM(instr->io.hdr.header_id))
7832                         break;
7833
7834                 if (i && data[i].n_users)
7835                         break;
7836         }
7837
7838         if (i < 2)
7839                 return 0;
7840
7841         *n_pattern_instr = i;
7842         return 1;
7843 }
7844
7845 static void
7846 instr_pattern_extract_many_optimize(struct instruction *instr,
7847                                     struct instruction_data *data,
7848                                     uint32_t n_instr)
7849 {
7850         uint32_t i;
7851
7852         for (i = 1; i < n_instr; i++) {
7853                 instr[0].type++;
7854                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7855                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7856                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7857
7858                 data[i].invalid = 1;
7859         }
7860 }
7861
7862 static int
7863 instr_pattern_emit_many_tx_detect(struct instruction *instr,
7864                                   struct instruction_data *data,
7865                                   uint32_t n_instr,
7866                                   uint32_t *n_pattern_instr)
7867 {
7868         uint32_t i;
7869
7870         for (i = 0; i < n_instr; i++) {
7871                 if (data[i].invalid)
7872                         break;
7873
7874                 if (instr[i].type != INSTR_HDR_EMIT)
7875                         break;
7876
7877                 if (i == RTE_DIM(instr->io.hdr.header_id))
7878                         break;
7879
7880                 if (i && data[i].n_users)
7881                         break;
7882         }
7883
7884         if (!i)
7885                 return 0;
7886
7887         if (!instruction_is_tx(instr[i].type))
7888                 return 0;
7889
7890         if (data[i].n_users)
7891                 return 0;
7892
7893         i++;
7894
7895         *n_pattern_instr = i;
7896         return 1;
7897 }
7898
7899 static void
7900 instr_pattern_emit_many_tx_optimize(struct instruction *instr,
7901                                     struct instruction_data *data,
7902                                     uint32_t n_instr)
7903 {
7904         uint32_t i;
7905
7906         /* Any emit instruction in addition to the first one. */
7907         for (i = 1; i < n_instr - 1; i++) {
7908                 instr[0].type++;
7909                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7910                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7911                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7912
7913                 data[i].invalid = 1;
7914         }
7915
7916         /* The TX instruction is the last one in the pattern. */
7917         instr[0].type++;
7918         instr[0].io.io.offset = instr[i].io.io.offset;
7919         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7920         data[i].invalid = 1;
7921 }
7922
7923 static int
7924 instr_pattern_dma_many_detect(struct instruction *instr,
7925                               struct instruction_data *data,
7926                               uint32_t n_instr,
7927                               uint32_t *n_pattern_instr)
7928 {
7929         uint32_t i;
7930
7931         for (i = 0; i < n_instr; i++) {
7932                 if (data[i].invalid)
7933                         break;
7934
7935                 if (instr[i].type != INSTR_DMA_HT)
7936                         break;
7937
7938                 if (i == RTE_DIM(instr->dma.dst.header_id))
7939                         break;
7940
7941                 if (i && data[i].n_users)
7942                         break;
7943         }
7944
7945         if (i < 2)
7946                 return 0;
7947
7948         *n_pattern_instr = i;
7949         return 1;
7950 }
7951
7952 static void
7953 instr_pattern_dma_many_optimize(struct instruction *instr,
7954                                 struct instruction_data *data,
7955                                 uint32_t n_instr)
7956 {
7957         uint32_t i;
7958
7959         for (i = 1; i < n_instr; i++) {
7960                 instr[0].type++;
7961                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7962                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7963                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7964                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7965
7966                 data[i].invalid = 1;
7967         }
7968 }
7969
7970 static uint32_t
7971 instr_optimize(struct instruction *instructions,
7972                struct instruction_data *instruction_data,
7973                uint32_t n_instructions)
7974 {
7975         uint32_t i, pos = 0;
7976
7977         for (i = 0; i < n_instructions; ) {
7978                 struct instruction *instr = &instructions[i];
7979                 struct instruction_data *data = &instruction_data[i];
7980                 uint32_t n_instr = 0;
7981                 int detected;
7982
7983                 /* Extract many. */
7984                 detected = instr_pattern_extract_many_detect(instr,
7985                                                              data,
7986                                                              n_instructions - i,
7987                                                              &n_instr);
7988                 if (detected) {
7989                         instr_pattern_extract_many_optimize(instr,
7990                                                             data,
7991                                                             n_instr);
7992                         i += n_instr;
7993                         continue;
7994                 }
7995
7996                 /* Emit many + TX. */
7997                 detected = instr_pattern_emit_many_tx_detect(instr,
7998                                                              data,
7999                                                              n_instructions - i,
8000                                                              &n_instr);
8001                 if (detected) {
8002                         instr_pattern_emit_many_tx_optimize(instr,
8003                                                             data,
8004                                                             n_instr);
8005                         i += n_instr;
8006                         continue;
8007                 }
8008
8009                 /* DMA many. */
8010                 detected = instr_pattern_dma_many_detect(instr,
8011                                                          data,
8012                                                          n_instructions - i,
8013                                                          &n_instr);
8014                 if (detected) {
8015                         instr_pattern_dma_many_optimize(instr, data, n_instr);
8016                         i += n_instr;
8017                         continue;
8018                 }
8019
8020                 /* No pattern starting at the current instruction. */
8021                 i++;
8022         }
8023
8024         /* Eliminate the invalid instructions that have been optimized out. */
8025         for (i = 0; i < n_instructions; i++) {
8026                 struct instruction *instr = &instructions[i];
8027                 struct instruction_data *data = &instruction_data[i];
8028
8029                 if (data->invalid)
8030                         continue;
8031
8032                 if (i != pos) {
8033                         memcpy(&instructions[pos], instr, sizeof(*instr));
8034                         memcpy(&instruction_data[pos], data, sizeof(*data));
8035                 }
8036
8037                 pos++;
8038         }
8039
8040         return pos;
8041 }
8042
8043 static int
8044 instruction_config(struct rte_swx_pipeline *p,
8045                    struct action *a,
8046                    const char **instructions,
8047                    uint32_t n_instructions)
8048 {
8049         struct instruction *instr = NULL;
8050         struct instruction_data *data = NULL;
8051         int err = 0;
8052         uint32_t i;
8053
8054         CHECK(n_instructions, EINVAL);
8055         CHECK(instructions, EINVAL);
8056         for (i = 0; i < n_instructions; i++)
8057                 CHECK_INSTRUCTION(instructions[i], EINVAL);
8058
8059         /* Memory allocation. */
8060         instr = calloc(n_instructions, sizeof(struct instruction));
8061         if (!instr) {
8062                 err = -ENOMEM;
8063                 goto error;
8064         }
8065
8066         data = calloc(n_instructions, sizeof(struct instruction_data));
8067         if (!data) {
8068                 err = -ENOMEM;
8069                 goto error;
8070         }
8071
8072         for (i = 0; i < n_instructions; i++) {
8073                 char *string = strdup(instructions[i]);
8074                 if (!string) {
8075                         err = -ENOMEM;
8076                         goto error;
8077                 }
8078
8079                 err = instr_translate(p, a, string, &instr[i], &data[i]);
8080                 if (err) {
8081                         free(string);
8082                         goto error;
8083                 }
8084
8085                 free(string);
8086         }
8087
8088         err = instr_label_check(data, n_instructions);
8089         if (err)
8090                 goto error;
8091
8092         err = instr_verify(p, a, instr, data, n_instructions);
8093         if (err)
8094                 goto error;
8095
8096         n_instructions = instr_optimize(instr, data, n_instructions);
8097
8098         err = instr_jmp_resolve(instr, data, n_instructions);
8099         if (err)
8100                 goto error;
8101
8102         if (a) {
8103                 a->instructions = instr;
8104                 a->n_instructions = n_instructions;
8105         } else {
8106                 p->instructions = instr;
8107                 p->n_instructions = n_instructions;
8108         }
8109
8110         free(data);
8111         return 0;
8112
8113 error:
8114         free(data);
8115         free(instr);
8116         return err;
8117 }
8118
8119 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
8120
8121 static instr_exec_t instruction_table[] = {
8122         [INSTR_RX] = instr_rx_exec,
8123         [INSTR_TX] = instr_tx_exec,
8124         [INSTR_TX_I] = instr_tx_i_exec,
8125
8126         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
8127         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
8128         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
8129         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
8130         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
8131         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
8132         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
8133         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
8134
8135         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
8136         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
8137         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
8138         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
8139         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
8140         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
8141         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
8142         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
8143         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
8144
8145         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
8146         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
8147
8148         [INSTR_MOV] = instr_mov_exec,
8149         [INSTR_MOV_S] = instr_mov_s_exec,
8150         [INSTR_MOV_I] = instr_mov_i_exec,
8151
8152         [INSTR_DMA_HT] = instr_dma_ht_exec,
8153         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
8154         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
8155         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
8156         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
8157         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
8158         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
8159         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
8160
8161         [INSTR_ALU_ADD] = instr_alu_add_exec,
8162         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
8163         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
8164         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
8165         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
8166         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
8167
8168         [INSTR_ALU_SUB] = instr_alu_sub_exec,
8169         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
8170         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
8171         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
8172         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
8173         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
8174
8175         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
8176         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
8177         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
8178         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
8179
8180         [INSTR_ALU_AND] = instr_alu_and_exec,
8181         [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
8182         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
8183
8184         [INSTR_ALU_OR] = instr_alu_or_exec,
8185         [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
8186         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
8187
8188         [INSTR_ALU_XOR] = instr_alu_xor_exec,
8189         [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec,
8190         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
8191
8192         [INSTR_ALU_SHL] = instr_alu_shl_exec,
8193         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
8194         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
8195         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
8196         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
8197         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
8198
8199         [INSTR_ALU_SHR] = instr_alu_shr_exec,
8200         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
8201         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
8202         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
8203         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
8204         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
8205
8206         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
8207         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
8208         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
8209
8210         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
8211         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
8212         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
8213         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
8214         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
8215         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
8216
8217         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
8218         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
8219         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
8220         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
8221         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
8222         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
8223         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
8224         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
8225         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
8226
8227         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
8228         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
8229         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
8230         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
8231         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
8232         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
8233         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
8234         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
8235         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
8236
8237         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
8238         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
8239         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
8240
8241         [INSTR_METER_HHM] = instr_meter_hhm_exec,
8242         [INSTR_METER_HHI] = instr_meter_hhi_exec,
8243         [INSTR_METER_HMM] = instr_meter_hmm_exec,
8244         [INSTR_METER_HMI] = instr_meter_hmi_exec,
8245         [INSTR_METER_MHM] = instr_meter_mhm_exec,
8246         [INSTR_METER_MHI] = instr_meter_mhi_exec,
8247         [INSTR_METER_MMM] = instr_meter_mmm_exec,
8248         [INSTR_METER_MMI] = instr_meter_mmi_exec,
8249         [INSTR_METER_IHM] = instr_meter_ihm_exec,
8250         [INSTR_METER_IHI] = instr_meter_ihi_exec,
8251         [INSTR_METER_IMM] = instr_meter_imm_exec,
8252         [INSTR_METER_IMI] = instr_meter_imi_exec,
8253
8254         [INSTR_TABLE] = instr_table_exec,
8255         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
8256         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
8257
8258         [INSTR_JMP] = instr_jmp_exec,
8259         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
8260         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
8261         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
8262         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
8263         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
8264         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
8265
8266         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
8267         [INSTR_JMP_EQ_S] = instr_jmp_eq_s_exec,
8268         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
8269
8270         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
8271         [INSTR_JMP_NEQ_S] = instr_jmp_neq_s_exec,
8272         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
8273
8274         [INSTR_JMP_LT] = instr_jmp_lt_exec,
8275         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
8276         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
8277         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
8278         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
8279         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
8280
8281         [INSTR_JMP_GT] = instr_jmp_gt_exec,
8282         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
8283         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
8284         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
8285         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
8286         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
8287
8288         [INSTR_RETURN] = instr_return_exec,
8289 };
8290
8291 static inline void
8292 instr_exec(struct rte_swx_pipeline *p)
8293 {
8294         struct thread *t = &p->threads[p->thread_id];
8295         struct instruction *ip = t->ip;
8296         instr_exec_t instr = instruction_table[ip->type];
8297
8298         instr(p);
8299 }
8300
8301 /*
8302  * Action.
8303  */
8304 static struct action *
8305 action_find(struct rte_swx_pipeline *p, const char *name)
8306 {
8307         struct action *elem;
8308
8309         if (!name)
8310                 return NULL;
8311
8312         TAILQ_FOREACH(elem, &p->actions, node)
8313                 if (strcmp(elem->name, name) == 0)
8314                         return elem;
8315
8316         return NULL;
8317 }
8318
8319 static struct action *
8320 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8321 {
8322         struct action *action = NULL;
8323
8324         TAILQ_FOREACH(action, &p->actions, node)
8325                 if (action->id == id)
8326                         return action;
8327
8328         return NULL;
8329 }
8330
8331 static struct field *
8332 action_field_find(struct action *a, const char *name)
8333 {
8334         return a->st ? struct_type_field_find(a->st, name) : NULL;
8335 }
8336
8337 static struct field *
8338 action_field_parse(struct action *action, const char *name)
8339 {
8340         if (name[0] != 't' || name[1] != '.')
8341                 return NULL;
8342
8343         return action_field_find(action, &name[2]);
8344 }
8345
8346 int
8347 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
8348                                const char *name,
8349                                const char *args_struct_type_name,
8350                                const char **instructions,
8351                                uint32_t n_instructions)
8352 {
8353         struct struct_type *args_struct_type;
8354         struct action *a;
8355         int err;
8356
8357         CHECK(p, EINVAL);
8358
8359         CHECK_NAME(name, EINVAL);
8360         CHECK(!action_find(p, name), EEXIST);
8361
8362         if (args_struct_type_name) {
8363                 CHECK_NAME(args_struct_type_name, EINVAL);
8364                 args_struct_type = struct_type_find(p, args_struct_type_name);
8365                 CHECK(args_struct_type, EINVAL);
8366         } else {
8367                 args_struct_type = NULL;
8368         }
8369
8370         /* Node allocation. */
8371         a = calloc(1, sizeof(struct action));
8372         CHECK(a, ENOMEM);
8373
8374         /* Node initialization. */
8375         strcpy(a->name, name);
8376         a->st = args_struct_type;
8377         a->id = p->n_actions;
8378
8379         /* Instruction translation. */
8380         err = instruction_config(p, a, instructions, n_instructions);
8381         if (err) {
8382                 free(a);
8383                 return err;
8384         }
8385
8386         /* Node add to tailq. */
8387         TAILQ_INSERT_TAIL(&p->actions, a, node);
8388         p->n_actions++;
8389
8390         return 0;
8391 }
8392
8393 static int
8394 action_build(struct rte_swx_pipeline *p)
8395 {
8396         struct action *action;
8397
8398         p->action_instructions = calloc(p->n_actions,
8399                                         sizeof(struct instruction *));
8400         CHECK(p->action_instructions, ENOMEM);
8401
8402         TAILQ_FOREACH(action, &p->actions, node)
8403                 p->action_instructions[action->id] = action->instructions;
8404
8405         return 0;
8406 }
8407
8408 static void
8409 action_build_free(struct rte_swx_pipeline *p)
8410 {
8411         free(p->action_instructions);
8412         p->action_instructions = NULL;
8413 }
8414
8415 static void
8416 action_free(struct rte_swx_pipeline *p)
8417 {
8418         action_build_free(p);
8419
8420         for ( ; ; ) {
8421                 struct action *action;
8422
8423                 action = TAILQ_FIRST(&p->actions);
8424                 if (!action)
8425                         break;
8426
8427                 TAILQ_REMOVE(&p->actions, action, node);
8428                 free(action->instructions);
8429                 free(action);
8430         }
8431 }
8432
8433 /*
8434  * Table.
8435  */
8436 static struct table_type *
8437 table_type_find(struct rte_swx_pipeline *p, const char *name)
8438 {
8439         struct table_type *elem;
8440
8441         TAILQ_FOREACH(elem, &p->table_types, node)
8442                 if (strcmp(elem->name, name) == 0)
8443                         return elem;
8444
8445         return NULL;
8446 }
8447
8448 static struct table_type *
8449 table_type_resolve(struct rte_swx_pipeline *p,
8450                    const char *recommended_type_name,
8451                    enum rte_swx_table_match_type match_type)
8452 {
8453         struct table_type *elem;
8454
8455         /* Only consider the recommended type if the match type is correct. */
8456         if (recommended_type_name)
8457                 TAILQ_FOREACH(elem, &p->table_types, node)
8458                         if (!strcmp(elem->name, recommended_type_name) &&
8459                             (elem->match_type == match_type))
8460                                 return elem;
8461
8462         /* Ignore the recommended type and get the first element with this match
8463          * type.
8464          */
8465         TAILQ_FOREACH(elem, &p->table_types, node)
8466                 if (elem->match_type == match_type)
8467                         return elem;
8468
8469         return NULL;
8470 }
8471
8472 static struct table *
8473 table_find(struct rte_swx_pipeline *p, const char *name)
8474 {
8475         struct table *elem;
8476
8477         TAILQ_FOREACH(elem, &p->tables, node)
8478                 if (strcmp(elem->name, name) == 0)
8479                         return elem;
8480
8481         return NULL;
8482 }
8483
8484 static struct table *
8485 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8486 {
8487         struct table *table = NULL;
8488
8489         TAILQ_FOREACH(table, &p->tables, node)
8490                 if (table->id == id)
8491                         return table;
8492
8493         return NULL;
8494 }
8495
8496 int
8497 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
8498                                      const char *name,
8499                                      enum rte_swx_table_match_type match_type,
8500                                      struct rte_swx_table_ops *ops)
8501 {
8502         struct table_type *elem;
8503
8504         CHECK(p, EINVAL);
8505
8506         CHECK_NAME(name, EINVAL);
8507         CHECK(!table_type_find(p, name), EEXIST);
8508
8509         CHECK(ops, EINVAL);
8510         CHECK(ops->create, EINVAL);
8511         CHECK(ops->lkp, EINVAL);
8512         CHECK(ops->free, EINVAL);
8513
8514         /* Node allocation. */
8515         elem = calloc(1, sizeof(struct table_type));
8516         CHECK(elem, ENOMEM);
8517
8518         /* Node initialization. */
8519         strcpy(elem->name, name);
8520         elem->match_type = match_type;
8521         memcpy(&elem->ops, ops, sizeof(*ops));
8522
8523         /* Node add to tailq. */
8524         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
8525
8526         return 0;
8527 }
8528
8529 static enum rte_swx_table_match_type
8530 table_match_type_resolve(struct rte_swx_match_field_params *fields,
8531                          uint32_t n_fields,
8532                          uint32_t max_offset_field_id)
8533 {
8534         uint32_t n_fields_em = 0, i;
8535
8536         for (i = 0; i < n_fields; i++)
8537                 if (fields[i].match_type == RTE_SWX_TABLE_MATCH_EXACT)
8538                         n_fields_em++;
8539
8540         if (n_fields_em == n_fields)
8541                 return RTE_SWX_TABLE_MATCH_EXACT;
8542
8543         if ((n_fields_em == n_fields - 1) &&
8544             (fields[max_offset_field_id].match_type == RTE_SWX_TABLE_MATCH_LPM))
8545                 return RTE_SWX_TABLE_MATCH_LPM;
8546
8547         return RTE_SWX_TABLE_MATCH_WILDCARD;
8548 }
8549
8550 static int
8551 table_match_fields_check(struct rte_swx_pipeline *p,
8552                          struct rte_swx_pipeline_table_params *params,
8553                          struct header **header,
8554                          uint32_t *min_offset_field_id,
8555                          uint32_t *max_offset_field_id)
8556 {
8557         struct header *h0 = NULL;
8558         struct field *hf, *mf;
8559         uint32_t *offset = NULL, min_offset, max_offset, min_offset_pos, max_offset_pos, i;
8560         int status = 0;
8561
8562         /* Return if no match fields. */
8563         if (!params->n_fields) {
8564                 if (params->fields) {
8565                         status = -EINVAL;
8566                         goto end;
8567                 }
8568
8569                 return 0;
8570         }
8571
8572         /* Memory allocation. */
8573         offset = calloc(params->n_fields, sizeof(uint32_t));
8574         if (!offset) {
8575                 status = -ENOMEM;
8576                 goto end;
8577         }
8578
8579         /* Check that all the match fields belong to either the same header or
8580          * to the meta-data.
8581          */
8582         hf = header_field_parse(p, params->fields[0].name, &h0);
8583         mf = metadata_field_parse(p, params->fields[0].name);
8584         if (!hf && !mf) {
8585                 status = -EINVAL;
8586                 goto end;
8587         }
8588
8589         offset[0] = h0 ? hf->offset : mf->offset;
8590
8591         for (i = 1; i < params->n_fields; i++)
8592                 if (h0) {
8593                         struct header *h;
8594
8595                         hf = header_field_parse(p, params->fields[i].name, &h);
8596                         if (!hf || (h->id != h0->id)) {
8597                                 status = -EINVAL;
8598                                 goto end;
8599                         }
8600
8601                         offset[i] = hf->offset;
8602                 } else {
8603                         mf = metadata_field_parse(p, params->fields[i].name);
8604                         if (!mf) {
8605                                 status = -EINVAL;
8606                                 goto end;
8607                         }
8608
8609                         offset[i] = mf->offset;
8610                 }
8611
8612         /* Check that there are no duplicated match fields. */
8613         for (i = 0; i < params->n_fields; i++) {
8614                 uint32_t j;
8615
8616                 for (j = 0; j < i; j++)
8617                         if (offset[j] == offset[i]) {
8618                                 status = -EINVAL;
8619                                 goto end;
8620                         }
8621         }
8622
8623         /* Find the min and max offset fields. */
8624         min_offset = offset[0];
8625         max_offset = offset[0];
8626         min_offset_pos = 0;
8627         max_offset_pos = 0;
8628
8629         for (i = 1; i < params->n_fields; i++) {
8630                 if (offset[i] < min_offset) {
8631                         min_offset = offset[i];
8632                         min_offset_pos = i;
8633                 }
8634
8635                 if (offset[i] > max_offset) {
8636                         max_offset = offset[i];
8637                         max_offset_pos = i;
8638                 }
8639         }
8640
8641         /* Return. */
8642         if (header)
8643                 *header = h0;
8644
8645         if (min_offset_field_id)
8646                 *min_offset_field_id = min_offset_pos;
8647
8648         if (max_offset_field_id)
8649                 *max_offset_field_id = max_offset_pos;
8650
8651 end:
8652         free(offset);
8653         return status;
8654 }
8655
8656 int
8657 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8658                               const char *name,
8659                               struct rte_swx_pipeline_table_params *params,
8660                               const char *recommended_table_type_name,
8661                               const char *args,
8662                               uint32_t size)
8663 {
8664         struct table_type *type;
8665         struct table *t;
8666         struct action *default_action;
8667         struct header *header = NULL;
8668         uint32_t action_data_size_max = 0, min_offset_field_id = 0, max_offset_field_id = 0, i;
8669         int status = 0;
8670
8671         CHECK(p, EINVAL);
8672
8673         CHECK_NAME(name, EINVAL);
8674         CHECK(!table_find(p, name), EEXIST);
8675
8676         CHECK(params, EINVAL);
8677
8678         /* Match checks. */
8679         status = table_match_fields_check(p,
8680                                           params,
8681                                           &header,
8682                                           &min_offset_field_id,
8683                                           &max_offset_field_id);
8684         if (status)
8685                 return status;
8686
8687         /* Action checks. */
8688         CHECK(params->n_actions, EINVAL);
8689         CHECK(params->action_names, EINVAL);
8690         for (i = 0; i < params->n_actions; i++) {
8691                 const char *action_name = params->action_names[i];
8692                 struct action *a;
8693                 uint32_t action_data_size;
8694
8695                 CHECK_NAME(action_name, EINVAL);
8696
8697                 a = action_find(p, action_name);
8698                 CHECK(a, EINVAL);
8699
8700                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8701                 if (action_data_size > action_data_size_max)
8702                         action_data_size_max = action_data_size;
8703         }
8704
8705         CHECK_NAME(params->default_action_name, EINVAL);
8706         for (i = 0; i < p->n_actions; i++)
8707                 if (!strcmp(params->action_names[i],
8708                             params->default_action_name))
8709                         break;
8710         CHECK(i < params->n_actions, EINVAL);
8711         default_action = action_find(p, params->default_action_name);
8712         CHECK((default_action->st && params->default_action_data) ||
8713               !params->default_action_data, EINVAL);
8714
8715         /* Table type checks. */
8716         if (recommended_table_type_name)
8717                 CHECK_NAME(recommended_table_type_name, EINVAL);
8718
8719         if (params->n_fields) {
8720                 enum rte_swx_table_match_type match_type;
8721
8722                 match_type = table_match_type_resolve(params->fields,
8723                                                       params->n_fields,
8724                                                       max_offset_field_id);
8725                 type = table_type_resolve(p,
8726                                           recommended_table_type_name,
8727                                           match_type);
8728                 CHECK(type, EINVAL);
8729         } else {
8730                 type = NULL;
8731         }
8732
8733         /* Memory allocation. */
8734         t = calloc(1, sizeof(struct table));
8735         CHECK(t, ENOMEM);
8736
8737         t->fields = calloc(params->n_fields, sizeof(struct match_field));
8738         if (!t->fields) {
8739                 free(t);
8740                 CHECK(0, ENOMEM);
8741         }
8742
8743         t->actions = calloc(params->n_actions, sizeof(struct action *));
8744         if (!t->actions) {
8745                 free(t->fields);
8746                 free(t);
8747                 CHECK(0, ENOMEM);
8748         }
8749
8750         if (action_data_size_max) {
8751                 t->default_action_data = calloc(1, action_data_size_max);
8752                 if (!t->default_action_data) {
8753                         free(t->actions);
8754                         free(t->fields);
8755                         free(t);
8756                         CHECK(0, ENOMEM);
8757                 }
8758         }
8759
8760         /* Node initialization. */
8761         strcpy(t->name, name);
8762         if (args && args[0])
8763                 strcpy(t->args, args);
8764         t->type = type;
8765
8766         for (i = 0; i < params->n_fields; i++) {
8767                 struct rte_swx_match_field_params *field = &params->fields[i];
8768                 struct match_field *f = &t->fields[i];
8769
8770                 f->match_type = field->match_type;
8771                 f->field = header ?
8772                         header_field_parse(p, field->name, NULL) :
8773                         metadata_field_parse(p, field->name);
8774         }
8775         t->n_fields = params->n_fields;
8776         t->header = header;
8777
8778         for (i = 0; i < params->n_actions; i++)
8779                 t->actions[i] = action_find(p, params->action_names[i]);
8780         t->default_action = default_action;
8781         if (default_action->st)
8782                 memcpy(t->default_action_data,
8783                        params->default_action_data,
8784                        default_action->st->n_bits / 8);
8785         t->n_actions = params->n_actions;
8786         t->default_action_is_const = params->default_action_is_const;
8787         t->action_data_size_max = action_data_size_max;
8788
8789         t->size = size;
8790         t->id = p->n_tables;
8791
8792         /* Node add to tailq. */
8793         TAILQ_INSERT_TAIL(&p->tables, t, node);
8794         p->n_tables++;
8795
8796         return 0;
8797 }
8798
8799 static struct rte_swx_table_params *
8800 table_params_get(struct table *table)
8801 {
8802         struct rte_swx_table_params *params;
8803         struct field *first, *last;
8804         uint8_t *key_mask;
8805         uint32_t key_size, key_offset, action_data_size, i;
8806
8807         /* Memory allocation. */
8808         params = calloc(1, sizeof(struct rte_swx_table_params));
8809         if (!params)
8810                 return NULL;
8811
8812         /* Find first (smallest offset) and last (biggest offset) match fields. */
8813         first = table->fields[0].field;
8814         last = table->fields[0].field;
8815
8816         for (i = 0; i < table->n_fields; i++) {
8817                 struct field *f = table->fields[i].field;
8818
8819                 if (f->offset < first->offset)
8820                         first = f;
8821
8822                 if (f->offset > last->offset)
8823                         last = f;
8824         }
8825
8826         /* Key offset and size. */
8827         key_offset = first->offset / 8;
8828         key_size = (last->offset + last->n_bits - first->offset) / 8;
8829
8830         /* Memory allocation. */
8831         key_mask = calloc(1, key_size);
8832         if (!key_mask) {
8833                 free(params);
8834                 return NULL;
8835         }
8836
8837         /* Key mask. */
8838         for (i = 0; i < table->n_fields; i++) {
8839                 struct field *f = table->fields[i].field;
8840                 uint32_t start = (f->offset - first->offset) / 8;
8841                 size_t size = f->n_bits / 8;
8842
8843                 memset(&key_mask[start], 0xFF, size);
8844         }
8845
8846         /* Action data size. */
8847         action_data_size = 0;
8848         for (i = 0; i < table->n_actions; i++) {
8849                 struct action *action = table->actions[i];
8850                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8851
8852                 if (ads > action_data_size)
8853                         action_data_size = ads;
8854         }
8855
8856         /* Fill in. */
8857         params->match_type = table->type->match_type;
8858         params->key_size = key_size;
8859         params->key_offset = key_offset;
8860         params->key_mask0 = key_mask;
8861         params->action_data_size = action_data_size;
8862         params->n_keys_max = table->size;
8863
8864         return params;
8865 }
8866
8867 static void
8868 table_params_free(struct rte_swx_table_params *params)
8869 {
8870         if (!params)
8871                 return;
8872
8873         free(params->key_mask0);
8874         free(params);
8875 }
8876
8877 static int
8878 table_state_build(struct rte_swx_pipeline *p)
8879 {
8880         struct table *table;
8881
8882         p->table_state = calloc(p->n_tables,
8883                                 sizeof(struct rte_swx_table_state));
8884         CHECK(p->table_state, ENOMEM);
8885
8886         TAILQ_FOREACH(table, &p->tables, node) {
8887                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8888
8889                 if (table->type) {
8890                         struct rte_swx_table_params *params;
8891
8892                         /* ts->obj. */
8893                         params = table_params_get(table);
8894                         CHECK(params, ENOMEM);
8895
8896                         ts->obj = table->type->ops.create(params,
8897                                 NULL,
8898                                 table->args,
8899                                 p->numa_node);
8900
8901                         table_params_free(params);
8902                         CHECK(ts->obj, ENODEV);
8903                 }
8904
8905                 /* ts->default_action_data. */
8906                 if (table->action_data_size_max) {
8907                         ts->default_action_data =
8908                                 malloc(table->action_data_size_max);
8909                         CHECK(ts->default_action_data, ENOMEM);
8910
8911                         memcpy(ts->default_action_data,
8912                                table->default_action_data,
8913                                table->action_data_size_max);
8914                 }
8915
8916                 /* ts->default_action_id. */
8917                 ts->default_action_id = table->default_action->id;
8918         }
8919
8920         return 0;
8921 }
8922
8923 static void
8924 table_state_build_free(struct rte_swx_pipeline *p)
8925 {
8926         uint32_t i;
8927
8928         if (!p->table_state)
8929                 return;
8930
8931         for (i = 0; i < p->n_tables; i++) {
8932                 struct rte_swx_table_state *ts = &p->table_state[i];
8933                 struct table *table = table_find_by_id(p, i);
8934
8935                 /* ts->obj. */
8936                 if (table->type && ts->obj)
8937                         table->type->ops.free(ts->obj);
8938
8939                 /* ts->default_action_data. */
8940                 free(ts->default_action_data);
8941         }
8942
8943         free(p->table_state);
8944         p->table_state = NULL;
8945 }
8946
8947 static void
8948 table_state_free(struct rte_swx_pipeline *p)
8949 {
8950         table_state_build_free(p);
8951 }
8952
8953 static int
8954 table_stub_lkp(void *table __rte_unused,
8955                void *mailbox __rte_unused,
8956                uint8_t **key __rte_unused,
8957                uint64_t *action_id __rte_unused,
8958                uint8_t **action_data __rte_unused,
8959                int *hit)
8960 {
8961         *hit = 0;
8962         return 1; /* DONE. */
8963 }
8964
8965 static int
8966 table_build(struct rte_swx_pipeline *p)
8967 {
8968         uint32_t i;
8969
8970         /* Per pipeline: table statistics. */
8971         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8972         CHECK(p->table_stats, ENOMEM);
8973
8974         for (i = 0; i < p->n_tables; i++) {
8975                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8976                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8977         }
8978
8979         /* Per thread: table runt-time. */
8980         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8981                 struct thread *t = &p->threads[i];
8982                 struct table *table;
8983
8984                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8985                 CHECK(t->tables, ENOMEM);
8986
8987                 TAILQ_FOREACH(table, &p->tables, node) {
8988                         struct table_runtime *r = &t->tables[table->id];
8989
8990                         if (table->type) {
8991                                 uint64_t size;
8992
8993                                 size = table->type->ops.mailbox_size_get();
8994
8995                                 /* r->func. */
8996                                 r->func = table->type->ops.lkp;
8997
8998                                 /* r->mailbox. */
8999                                 if (size) {
9000                                         r->mailbox = calloc(1, size);
9001                                         CHECK(r->mailbox, ENOMEM);
9002                                 }
9003
9004                                 /* r->key. */
9005                                 r->key = table->header ?
9006                                         &t->structs[table->header->struct_id] :
9007                                         &t->structs[p->metadata_struct_id];
9008                         } else {
9009                                 r->func = table_stub_lkp;
9010                         }
9011                 }
9012         }
9013
9014         return 0;
9015 }
9016
9017 static void
9018 table_build_free(struct rte_swx_pipeline *p)
9019 {
9020         uint32_t i;
9021
9022         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9023                 struct thread *t = &p->threads[i];
9024                 uint32_t j;
9025
9026                 if (!t->tables)
9027                         continue;
9028
9029                 for (j = 0; j < p->n_tables; j++) {
9030                         struct table_runtime *r = &t->tables[j];
9031
9032                         free(r->mailbox);
9033                 }
9034
9035                 free(t->tables);
9036                 t->tables = NULL;
9037         }
9038
9039         if (p->table_stats) {
9040                 for (i = 0; i < p->n_tables; i++)
9041                         free(p->table_stats[i].n_pkts_action);
9042
9043                 free(p->table_stats);
9044         }
9045 }
9046
9047 static void
9048 table_free(struct rte_swx_pipeline *p)
9049 {
9050         table_build_free(p);
9051
9052         /* Tables. */
9053         for ( ; ; ) {
9054                 struct table *elem;
9055
9056                 elem = TAILQ_FIRST(&p->tables);
9057                 if (!elem)
9058                         break;
9059
9060                 TAILQ_REMOVE(&p->tables, elem, node);
9061                 free(elem->fields);
9062                 free(elem->actions);
9063                 free(elem->default_action_data);
9064                 free(elem);
9065         }
9066
9067         /* Table types. */
9068         for ( ; ; ) {
9069                 struct table_type *elem;
9070
9071                 elem = TAILQ_FIRST(&p->table_types);
9072                 if (!elem)
9073                         break;
9074
9075                 TAILQ_REMOVE(&p->table_types, elem, node);
9076                 free(elem);
9077         }
9078 }
9079
9080 /*
9081  * Register array.
9082  */
9083 static struct regarray *
9084 regarray_find(struct rte_swx_pipeline *p, const char *name)
9085 {
9086         struct regarray *elem;
9087
9088         TAILQ_FOREACH(elem, &p->regarrays, node)
9089                 if (!strcmp(elem->name, name))
9090                         return elem;
9091
9092         return NULL;
9093 }
9094
9095 static struct regarray *
9096 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9097 {
9098         struct regarray *elem = NULL;
9099
9100         TAILQ_FOREACH(elem, &p->regarrays, node)
9101                 if (elem->id == id)
9102                         return elem;
9103
9104         return NULL;
9105 }
9106
9107 int
9108 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9109                               const char *name,
9110                               uint32_t size,
9111                               uint64_t init_val)
9112 {
9113         struct regarray *r;
9114
9115         CHECK(p, EINVAL);
9116
9117         CHECK_NAME(name, EINVAL);
9118         CHECK(!regarray_find(p, name), EEXIST);
9119
9120         CHECK(size, EINVAL);
9121         size = rte_align32pow2(size);
9122
9123         /* Memory allocation. */
9124         r = calloc(1, sizeof(struct regarray));
9125         CHECK(r, ENOMEM);
9126
9127         /* Node initialization. */
9128         strcpy(r->name, name);
9129         r->init_val = init_val;
9130         r->size = size;
9131         r->id = p->n_regarrays;
9132
9133         /* Node add to tailq. */
9134         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9135         p->n_regarrays++;
9136
9137         return 0;
9138 }
9139
9140 static int
9141 regarray_build(struct rte_swx_pipeline *p)
9142 {
9143         struct regarray *regarray;
9144
9145         if (!p->n_regarrays)
9146                 return 0;
9147
9148         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9149         CHECK(p->regarray_runtime, ENOMEM);
9150
9151         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9152                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9153                 uint32_t i;
9154
9155                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9156                                          RTE_CACHE_LINE_SIZE,
9157                                          p->numa_node);
9158                 CHECK(r->regarray, ENOMEM);
9159
9160                 if (regarray->init_val)
9161                         for (i = 0; i < regarray->size; i++)
9162                                 r->regarray[i] = regarray->init_val;
9163
9164                 r->size_mask = regarray->size - 1;
9165         }
9166
9167         return 0;
9168 }
9169
9170 static void
9171 regarray_build_free(struct rte_swx_pipeline *p)
9172 {
9173         uint32_t i;
9174
9175         if (!p->regarray_runtime)
9176                 return;
9177
9178         for (i = 0; i < p->n_regarrays; i++) {
9179                 struct regarray *regarray = regarray_find_by_id(p, i);
9180                 struct regarray_runtime *r = &p->regarray_runtime[i];
9181
9182                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9183         }
9184
9185         free(p->regarray_runtime);
9186         p->regarray_runtime = NULL;
9187 }
9188
9189 static void
9190 regarray_free(struct rte_swx_pipeline *p)
9191 {
9192         regarray_build_free(p);
9193
9194         for ( ; ; ) {
9195                 struct regarray *elem;
9196
9197                 elem = TAILQ_FIRST(&p->regarrays);
9198                 if (!elem)
9199                         break;
9200
9201                 TAILQ_REMOVE(&p->regarrays, elem, node);
9202                 free(elem);
9203         }
9204 }
9205
9206 /*
9207  * Meter array.
9208  */
9209 static struct meter_profile *
9210 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9211 {
9212         struct meter_profile *elem;
9213
9214         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9215                 if (!strcmp(elem->name, name))
9216                         return elem;
9217
9218         return NULL;
9219 }
9220
9221 static struct metarray *
9222 metarray_find(struct rte_swx_pipeline *p, const char *name)
9223 {
9224         struct metarray *elem;
9225
9226         TAILQ_FOREACH(elem, &p->metarrays, node)
9227                 if (!strcmp(elem->name, name))
9228                         return elem;
9229
9230         return NULL;
9231 }
9232
9233 static struct metarray *
9234 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9235 {
9236         struct metarray *elem = NULL;
9237
9238         TAILQ_FOREACH(elem, &p->metarrays, node)
9239                 if (elem->id == id)
9240                         return elem;
9241
9242         return NULL;
9243 }
9244
9245 int
9246 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9247                                  const char *name,
9248                                  uint32_t size)
9249 {
9250         struct metarray *m;
9251
9252         CHECK(p, EINVAL);
9253
9254         CHECK_NAME(name, EINVAL);
9255         CHECK(!metarray_find(p, name), EEXIST);
9256
9257         CHECK(size, EINVAL);
9258         size = rte_align32pow2(size);
9259
9260         /* Memory allocation. */
9261         m = calloc(1, sizeof(struct metarray));
9262         CHECK(m, ENOMEM);
9263
9264         /* Node initialization. */
9265         strcpy(m->name, name);
9266         m->size = size;
9267         m->id = p->n_metarrays;
9268
9269         /* Node add to tailq. */
9270         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9271         p->n_metarrays++;
9272
9273         return 0;
9274 }
9275
9276 struct meter_profile meter_profile_default = {
9277         .node = {0},
9278         .name = "",
9279         .params = {0},
9280
9281         .profile = {
9282                 .cbs = 10000,
9283                 .pbs = 10000,
9284                 .cir_period = 1,
9285                 .cir_bytes_per_period = 1,
9286                 .pir_period = 1,
9287                 .pir_bytes_per_period = 1,
9288         },
9289
9290         .n_users = 0,
9291 };
9292
9293 static void
9294 meter_init(struct meter *m)
9295 {
9296         memset(m, 0, sizeof(struct meter));
9297         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9298         m->profile = &meter_profile_default;
9299         m->color_mask = RTE_COLOR_GREEN;
9300
9301         meter_profile_default.n_users++;
9302 }
9303
9304 static int
9305 metarray_build(struct rte_swx_pipeline *p)
9306 {
9307         struct metarray *m;
9308
9309         if (!p->n_metarrays)
9310                 return 0;
9311
9312         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9313         CHECK(p->metarray_runtime, ENOMEM);
9314
9315         TAILQ_FOREACH(m, &p->metarrays, node) {
9316                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9317                 uint32_t i;
9318
9319                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9320                                          RTE_CACHE_LINE_SIZE,
9321                                          p->numa_node);
9322                 CHECK(r->metarray, ENOMEM);
9323
9324                 for (i = 0; i < m->size; i++)
9325                         meter_init(&r->metarray[i]);
9326
9327                 r->size_mask = m->size - 1;
9328         }
9329
9330         return 0;
9331 }
9332
9333 static void
9334 metarray_build_free(struct rte_swx_pipeline *p)
9335 {
9336         uint32_t i;
9337
9338         if (!p->metarray_runtime)
9339                 return;
9340
9341         for (i = 0; i < p->n_metarrays; i++) {
9342                 struct metarray *m = metarray_find_by_id(p, i);
9343                 struct metarray_runtime *r = &p->metarray_runtime[i];
9344
9345                 env_free(r->metarray, m->size * sizeof(struct meter));
9346         }
9347
9348         free(p->metarray_runtime);
9349         p->metarray_runtime = NULL;
9350 }
9351
9352 static void
9353 metarray_free(struct rte_swx_pipeline *p)
9354 {
9355         metarray_build_free(p);
9356
9357         /* Meter arrays. */
9358         for ( ; ; ) {
9359                 struct metarray *elem;
9360
9361                 elem = TAILQ_FIRST(&p->metarrays);
9362                 if (!elem)
9363                         break;
9364
9365                 TAILQ_REMOVE(&p->metarrays, elem, node);
9366                 free(elem);
9367         }
9368
9369         /* Meter profiles. */
9370         for ( ; ; ) {
9371                 struct meter_profile *elem;
9372
9373                 elem = TAILQ_FIRST(&p->meter_profiles);
9374                 if (!elem)
9375                         break;
9376
9377                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9378                 free(elem);
9379         }
9380 }
9381
9382 /*
9383  * Pipeline.
9384  */
9385 int
9386 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9387 {
9388         struct rte_swx_pipeline *pipeline;
9389
9390         /* Check input parameters. */
9391         CHECK(p, EINVAL);
9392
9393         /* Memory allocation. */
9394         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9395         CHECK(pipeline, ENOMEM);
9396
9397         /* Initialization. */
9398         TAILQ_INIT(&pipeline->struct_types);
9399         TAILQ_INIT(&pipeline->port_in_types);
9400         TAILQ_INIT(&pipeline->ports_in);
9401         TAILQ_INIT(&pipeline->port_out_types);
9402         TAILQ_INIT(&pipeline->ports_out);
9403         TAILQ_INIT(&pipeline->extern_types);
9404         TAILQ_INIT(&pipeline->extern_objs);
9405         TAILQ_INIT(&pipeline->extern_funcs);
9406         TAILQ_INIT(&pipeline->headers);
9407         TAILQ_INIT(&pipeline->actions);
9408         TAILQ_INIT(&pipeline->table_types);
9409         TAILQ_INIT(&pipeline->tables);
9410         TAILQ_INIT(&pipeline->regarrays);
9411         TAILQ_INIT(&pipeline->meter_profiles);
9412         TAILQ_INIT(&pipeline->metarrays);
9413
9414         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9415         pipeline->numa_node = numa_node;
9416
9417         *p = pipeline;
9418         return 0;
9419 }
9420
9421 void
9422 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9423 {
9424         if (!p)
9425                 return;
9426
9427         free(p->instructions);
9428
9429         metarray_free(p);
9430         regarray_free(p);
9431         table_state_free(p);
9432         table_free(p);
9433         action_free(p);
9434         metadata_free(p);
9435         header_free(p);
9436         extern_func_free(p);
9437         extern_obj_free(p);
9438         port_out_free(p);
9439         port_in_free(p);
9440         struct_free(p);
9441
9442         free(p);
9443 }
9444
9445 int
9446 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9447                                      const char **instructions,
9448                                      uint32_t n_instructions)
9449 {
9450         int err;
9451         uint32_t i;
9452
9453         err = instruction_config(p, NULL, instructions, n_instructions);
9454         if (err)
9455                 return err;
9456
9457         /* Thread instruction pointer reset. */
9458         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9459                 struct thread *t = &p->threads[i];
9460
9461                 thread_ip_reset(p, t);
9462         }
9463
9464         return 0;
9465 }
9466
9467 int
9468 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9469 {
9470         int status;
9471
9472         CHECK(p, EINVAL);
9473         CHECK(p->build_done == 0, EEXIST);
9474
9475         status = port_in_build(p);
9476         if (status)
9477                 goto error;
9478
9479         status = port_out_build(p);
9480         if (status)
9481                 goto error;
9482
9483         status = struct_build(p);
9484         if (status)
9485                 goto error;
9486
9487         status = extern_obj_build(p);
9488         if (status)
9489                 goto error;
9490
9491         status = extern_func_build(p);
9492         if (status)
9493                 goto error;
9494
9495         status = header_build(p);
9496         if (status)
9497                 goto error;
9498
9499         status = metadata_build(p);
9500         if (status)
9501                 goto error;
9502
9503         status = action_build(p);
9504         if (status)
9505                 goto error;
9506
9507         status = table_build(p);
9508         if (status)
9509                 goto error;
9510
9511         status = table_state_build(p);
9512         if (status)
9513                 goto error;
9514
9515         status = regarray_build(p);
9516         if (status)
9517                 goto error;
9518
9519         status = metarray_build(p);
9520         if (status)
9521                 goto error;
9522
9523         p->build_done = 1;
9524         return 0;
9525
9526 error:
9527         metarray_build_free(p);
9528         regarray_build_free(p);
9529         table_state_build_free(p);
9530         table_build_free(p);
9531         action_build_free(p);
9532         metadata_build_free(p);
9533         header_build_free(p);
9534         extern_func_build_free(p);
9535         extern_obj_build_free(p);
9536         port_out_build_free(p);
9537         port_in_build_free(p);
9538         struct_build_free(p);
9539
9540         return status;
9541 }
9542
9543 void
9544 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9545 {
9546         uint32_t i;
9547
9548         for (i = 0; i < n_instructions; i++)
9549                 instr_exec(p);
9550 }
9551
9552 void
9553 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9554 {
9555         uint32_t i;
9556
9557         for (i = 0; i < p->n_ports_out; i++) {
9558                 struct port_out_runtime *port = &p->out[i];
9559
9560                 if (port->flush)
9561                         port->flush(port->obj);
9562         }
9563 }
9564
9565 /*
9566  * Control.
9567  */
9568 int
9569 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9570                               struct rte_swx_ctl_pipeline_info *pipeline)
9571 {
9572         struct action *action;
9573         struct table *table;
9574         uint32_t n_actions = 0, n_tables = 0;
9575
9576         if (!p || !pipeline)
9577                 return -EINVAL;
9578
9579         TAILQ_FOREACH(action, &p->actions, node)
9580                 n_actions++;
9581
9582         TAILQ_FOREACH(table, &p->tables, node)
9583                 n_tables++;
9584
9585         pipeline->n_ports_in = p->n_ports_in;
9586         pipeline->n_ports_out = p->n_ports_out;
9587         pipeline->n_actions = n_actions;
9588         pipeline->n_tables = n_tables;
9589         pipeline->n_regarrays = p->n_regarrays;
9590         pipeline->n_metarrays = p->n_metarrays;
9591
9592         return 0;
9593 }
9594
9595 int
9596 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9597 {
9598         if (!p || !numa_node)
9599                 return -EINVAL;
9600
9601         *numa_node = p->numa_node;
9602         return 0;
9603 }
9604
9605 int
9606 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9607                             uint32_t action_id,
9608                             struct rte_swx_ctl_action_info *action)
9609 {
9610         struct action *a = NULL;
9611
9612         if (!p || (action_id >= p->n_actions) || !action)
9613                 return -EINVAL;
9614
9615         a = action_find_by_id(p, action_id);
9616         if (!a)
9617                 return -EINVAL;
9618
9619         strcpy(action->name, a->name);
9620         action->n_args = a->st ? a->st->n_fields : 0;
9621         return 0;
9622 }
9623
9624 int
9625 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9626                                 uint32_t action_id,
9627                                 uint32_t action_arg_id,
9628                                 struct rte_swx_ctl_action_arg_info *action_arg)
9629 {
9630         struct action *a = NULL;
9631         struct field *arg = NULL;
9632
9633         if (!p || (action_id >= p->n_actions) || !action_arg)
9634                 return -EINVAL;
9635
9636         a = action_find_by_id(p, action_id);
9637         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9638                 return -EINVAL;
9639
9640         arg = &a->st->fields[action_arg_id];
9641         strcpy(action_arg->name, arg->name);
9642         action_arg->n_bits = arg->n_bits;
9643
9644         return 0;
9645 }
9646
9647 int
9648 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9649                            uint32_t table_id,
9650                            struct rte_swx_ctl_table_info *table)
9651 {
9652         struct table *t = NULL;
9653
9654         if (!p || !table)
9655                 return -EINVAL;
9656
9657         t = table_find_by_id(p, table_id);
9658         if (!t)
9659                 return -EINVAL;
9660
9661         strcpy(table->name, t->name);
9662         strcpy(table->args, t->args);
9663         table->n_match_fields = t->n_fields;
9664         table->n_actions = t->n_actions;
9665         table->default_action_is_const = t->default_action_is_const;
9666         table->size = t->size;
9667         return 0;
9668 }
9669
9670 int
9671 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9672         uint32_t table_id,
9673         uint32_t match_field_id,
9674         struct rte_swx_ctl_table_match_field_info *match_field)
9675 {
9676         struct table *t;
9677         struct match_field *f;
9678
9679         if (!p || (table_id >= p->n_tables) || !match_field)
9680                 return -EINVAL;
9681
9682         t = table_find_by_id(p, table_id);
9683         if (!t || (match_field_id >= t->n_fields))
9684                 return -EINVAL;
9685
9686         f = &t->fields[match_field_id];
9687         match_field->match_type = f->match_type;
9688         match_field->is_header = t->header ? 1 : 0;
9689         match_field->n_bits = f->field->n_bits;
9690         match_field->offset = f->field->offset;
9691
9692         return 0;
9693 }
9694
9695 int
9696 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9697         uint32_t table_id,
9698         uint32_t table_action_id,
9699         struct rte_swx_ctl_table_action_info *table_action)
9700 {
9701         struct table *t;
9702
9703         if (!p || (table_id >= p->n_tables) || !table_action)
9704                 return -EINVAL;
9705
9706         t = table_find_by_id(p, table_id);
9707         if (!t || (table_action_id >= t->n_actions))
9708                 return -EINVAL;
9709
9710         table_action->action_id = t->actions[table_action_id]->id;
9711
9712         return 0;
9713 }
9714
9715 int
9716 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9717                           uint32_t table_id,
9718                           struct rte_swx_table_ops *table_ops,
9719                           int *is_stub)
9720 {
9721         struct table *t;
9722
9723         if (!p || (table_id >= p->n_tables))
9724                 return -EINVAL;
9725
9726         t = table_find_by_id(p, table_id);
9727         if (!t)
9728                 return -EINVAL;
9729
9730         if (t->type) {
9731                 if (table_ops)
9732                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9733                 *is_stub = 0;
9734         } else {
9735                 *is_stub = 1;
9736         }
9737
9738         return 0;
9739 }
9740
9741 int
9742 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9743                                  struct rte_swx_table_state **table_state)
9744 {
9745         if (!p || !table_state || !p->build_done)
9746                 return -EINVAL;
9747
9748         *table_state = p->table_state;
9749         return 0;
9750 }
9751
9752 int
9753 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9754                                  struct rte_swx_table_state *table_state)
9755 {
9756         if (!p || !table_state || !p->build_done)
9757                 return -EINVAL;
9758
9759         p->table_state = table_state;
9760         return 0;
9761 }
9762
9763 int
9764 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9765                                         uint32_t port_id,
9766                                         struct rte_swx_port_in_stats *stats)
9767 {
9768         struct port_in *port;
9769
9770         if (!p || !stats)
9771                 return -EINVAL;
9772
9773         port = port_in_find(p, port_id);
9774         if (!port)
9775                 return -EINVAL;
9776
9777         port->type->ops.stats_read(port->obj, stats);
9778         return 0;
9779 }
9780
9781 int
9782 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9783                                          uint32_t port_id,
9784                                          struct rte_swx_port_out_stats *stats)
9785 {
9786         struct port_out *port;
9787
9788         if (!p || !stats)
9789                 return -EINVAL;
9790
9791         port = port_out_find(p, port_id);
9792         if (!port)
9793                 return -EINVAL;
9794
9795         port->type->ops.stats_read(port->obj, stats);
9796         return 0;
9797 }
9798
9799 int
9800 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9801                                       const char *table_name,
9802                                       struct rte_swx_table_stats *stats)
9803 {
9804         struct table *table;
9805         struct table_statistics *table_stats;
9806
9807         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9808                 return -EINVAL;
9809
9810         table = table_find(p, table_name);
9811         if (!table)
9812                 return -EINVAL;
9813
9814         table_stats = &p->table_stats[table->id];
9815
9816         memcpy(&stats->n_pkts_action,
9817                &table_stats->n_pkts_action,
9818                p->n_actions * sizeof(uint64_t));
9819
9820         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9821         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9822
9823         return 0;
9824 }
9825
9826 int
9827 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9828                               uint32_t regarray_id,
9829                               struct rte_swx_ctl_regarray_info *regarray)
9830 {
9831         struct regarray *r;
9832
9833         if (!p || !regarray)
9834                 return -EINVAL;
9835
9836         r = regarray_find_by_id(p, regarray_id);
9837         if (!r)
9838                 return -EINVAL;
9839
9840         strcpy(regarray->name, r->name);
9841         regarray->size = r->size;
9842         return 0;
9843 }
9844
9845 int
9846 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9847                                    const char *regarray_name,
9848                                    uint32_t regarray_index,
9849                                    uint64_t *value)
9850 {
9851         struct regarray *regarray;
9852         struct regarray_runtime *r;
9853
9854         if (!p || !regarray_name || !value)
9855                 return -EINVAL;
9856
9857         regarray = regarray_find(p, regarray_name);
9858         if (!regarray || (regarray_index >= regarray->size))
9859                 return -EINVAL;
9860
9861         r = &p->regarray_runtime[regarray->id];
9862         *value = r->regarray[regarray_index];
9863         return 0;
9864 }
9865
9866 int
9867 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9868                                    const char *regarray_name,
9869                                    uint32_t regarray_index,
9870                                    uint64_t value)
9871 {
9872         struct regarray *regarray;
9873         struct regarray_runtime *r;
9874
9875         if (!p || !regarray_name)
9876                 return -EINVAL;
9877
9878         regarray = regarray_find(p, regarray_name);
9879         if (!regarray || (regarray_index >= regarray->size))
9880                 return -EINVAL;
9881
9882         r = &p->regarray_runtime[regarray->id];
9883         r->regarray[regarray_index] = value;
9884         return 0;
9885 }
9886
9887 int
9888 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9889                               uint32_t metarray_id,
9890                               struct rte_swx_ctl_metarray_info *metarray)
9891 {
9892         struct metarray *m;
9893
9894         if (!p || !metarray)
9895                 return -EINVAL;
9896
9897         m = metarray_find_by_id(p, metarray_id);
9898         if (!m)
9899                 return -EINVAL;
9900
9901         strcpy(metarray->name, m->name);
9902         metarray->size = m->size;
9903         return 0;
9904 }
9905
9906 int
9907 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9908                               const char *name,
9909                               struct rte_meter_trtcm_params *params)
9910 {
9911         struct meter_profile *mp;
9912         int status;
9913
9914         CHECK(p, EINVAL);
9915         CHECK_NAME(name, EINVAL);
9916         CHECK(params, EINVAL);
9917         CHECK(!meter_profile_find(p, name), EEXIST);
9918
9919         /* Node allocation. */
9920         mp = calloc(1, sizeof(struct meter_profile));
9921         CHECK(mp, ENOMEM);
9922
9923         /* Node initialization. */
9924         strcpy(mp->name, name);
9925         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9926         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9927         if (status) {
9928                 free(mp);
9929                 CHECK(0, EINVAL);
9930         }
9931
9932         /* Node add to tailq. */
9933         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9934
9935         return 0;
9936 }
9937
9938 int
9939 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9940                                  const char *name)
9941 {
9942         struct meter_profile *mp;
9943
9944         CHECK(p, EINVAL);
9945         CHECK_NAME(name, EINVAL);
9946
9947         mp = meter_profile_find(p, name);
9948         CHECK(mp, EINVAL);
9949         CHECK(!mp->n_users, EBUSY);
9950
9951         /* Remove node from tailq. */
9952         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9953         free(mp);
9954
9955         return 0;
9956 }
9957
9958 int
9959 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9960                         const char *metarray_name,
9961                         uint32_t metarray_index)
9962 {
9963         struct meter_profile *mp_old;
9964         struct metarray *metarray;
9965         struct metarray_runtime *metarray_runtime;
9966         struct meter *m;
9967
9968         CHECK(p, EINVAL);
9969         CHECK_NAME(metarray_name, EINVAL);
9970
9971         metarray = metarray_find(p, metarray_name);
9972         CHECK(metarray, EINVAL);
9973         CHECK(metarray_index < metarray->size, EINVAL);
9974
9975         metarray_runtime = &p->metarray_runtime[metarray->id];
9976         m = &metarray_runtime->metarray[metarray_index];
9977         mp_old = m->profile;
9978
9979         meter_init(m);
9980
9981         mp_old->n_users--;
9982
9983         return 0;
9984 }
9985
9986 int
9987 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9988                       const char *metarray_name,
9989                       uint32_t metarray_index,
9990                       const char *profile_name)
9991 {
9992         struct meter_profile *mp, *mp_old;
9993         struct metarray *metarray;
9994         struct metarray_runtime *metarray_runtime;
9995         struct meter *m;
9996
9997         CHECK(p, EINVAL);
9998         CHECK_NAME(metarray_name, EINVAL);
9999
10000         metarray = metarray_find(p, metarray_name);
10001         CHECK(metarray, EINVAL);
10002         CHECK(metarray_index < metarray->size, EINVAL);
10003
10004         mp = meter_profile_find(p, profile_name);
10005         CHECK(mp, EINVAL);
10006
10007         metarray_runtime = &p->metarray_runtime[metarray->id];
10008         m = &metarray_runtime->metarray[metarray_index];
10009         mp_old = m->profile;
10010
10011         memset(m, 0, sizeof(struct meter));
10012         rte_meter_trtcm_config(&m->m, &mp->profile);
10013         m->profile = mp;
10014         m->color_mask = RTE_COLORS;
10015
10016         mp->n_users++;
10017         mp_old->n_users--;
10018
10019         return 0;
10020 }
10021
10022 int
10023 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10024                              const char *metarray_name,
10025                              uint32_t metarray_index,
10026                              struct rte_swx_ctl_meter_stats *stats)
10027 {
10028         struct metarray *metarray;
10029         struct metarray_runtime *metarray_runtime;
10030         struct meter *m;
10031
10032         CHECK(p, EINVAL);
10033         CHECK_NAME(metarray_name, EINVAL);
10034
10035         metarray = metarray_find(p, metarray_name);
10036         CHECK(metarray, EINVAL);
10037         CHECK(metarray_index < metarray->size, EINVAL);
10038
10039         CHECK(stats, EINVAL);
10040
10041         metarray_runtime = &p->metarray_runtime[metarray->id];
10042         m = &metarray_runtime->metarray[metarray_index];
10043
10044         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10045         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10046
10047         return 0;
10048 }