lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <inttypes.h>
   9 #include <sys/queue.h>
  10 #include <arpa/inet.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_prefetch.h>
  14 #include <rte_byteorder.h>
  15 #include <rte_cycles.h>
  16 #include <rte_meter.h>
  17
  18 #include "rte_swx_pipeline.h"
  19 #include "rte_swx_ctl.h"
  20
  21 #define CHECK(condition, err_code)                                             \
  22 do {                                                                           \
  23         if (!(condition))                                                      \
  24                 return -(err_code);                                            \
  25 } while (0)
  26
  27 #define CHECK_NAME(name, err_code)                                             \
  28         CHECK((name) &&                                                        \
  29               (name)[0] &&                                                     \
  30               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  31               err_code)
  32
  33 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  34         CHECK((instr) &&                                                       \
  35               (instr)[0] &&                                                    \
  36               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  37                RTE_SWX_INSTRUCTION_SIZE),                                      \
  38               err_code)
  39
  40 #ifndef TRACE_LEVEL
  41 #define TRACE_LEVEL 0
  42 #endif
  43
  44 #if TRACE_LEVEL
  45 #define TRACE(...) printf(__VA_ARGS__)
  46 #else
  47 #define TRACE(...)
  48 #endif
  49
  50 /*
  51  * Environment.
  52  */
  53 #define ntoh64(x) rte_be_to_cpu_64(x)
  54 #define hton64(x) rte_cpu_to_be_64(x)
  55
  56 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  57
  58 #include <rte_malloc.h>
  59
  60 static void *
  61 env_malloc(size_t size, size_t alignment, int numa_node)
  62 {
  63         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  64 }
  65
  66 static void
  67 env_free(void *start, size_t size __rte_unused)
  68 {
  69         rte_free(start);
  70 }
  71
  72 #else
  73
  74 #include <numa.h>
  75
  76 static void *
  77 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  78 {
  79         void *start;
  80
  81         if (numa_available() == -1)
  82                 return NULL;
  83
  84         start = numa_alloc_onnode(size, numa_node);
  85         if (!start)
  86                 return NULL;
  87
  88         memset(start, 0, size);
  89         return start;
  90 }
  91
  92 static void
  93 env_free(void *start, size_t size)
  94 {
  95         if (numa_available() == -1)
  96                 return;
  97
  98         numa_free(start, size);
  99 }
 100
 101 #endif
 102
 103 /*
 104  * Struct.
 105  */
 106 struct field {
 107         char name[RTE_SWX_NAME_SIZE];
 108         uint32_t n_bits;
 109         uint32_t offset;
 110 };
 111
 112 struct struct_type {
 113         TAILQ_ENTRY(struct_type) node;
 114         char name[RTE_SWX_NAME_SIZE];
 115         struct field *fields;
 116         uint32_t n_fields;
 117         uint32_t n_bits;
 118 };
 119
 120 TAILQ_HEAD(struct_type_tailq, struct_type);
 121
 122 /*
 123  * Input port.
 124  */
 125 struct port_in_type {
 126         TAILQ_ENTRY(port_in_type) node;
 127         char name[RTE_SWX_NAME_SIZE];
 128         struct rte_swx_port_in_ops ops;
 129 };
 130
 131 TAILQ_HEAD(port_in_type_tailq, port_in_type);
 132
 133 struct port_in {
 134         TAILQ_ENTRY(port_in) node;
 135         struct port_in_type *type;
 136         void *obj;
 137         uint32_t id;
 138 };
 139
 140 TAILQ_HEAD(port_in_tailq, port_in);
 141
 142 struct port_in_runtime {
 143         rte_swx_port_in_pkt_rx_t pkt_rx;
 144         void *obj;
 145 };
 146
 147 /*
 148  * Output port.
 149  */
 150 struct port_out_type {
 151         TAILQ_ENTRY(port_out_type) node;
 152         char name[RTE_SWX_NAME_SIZE];
 153         struct rte_swx_port_out_ops ops;
 154 };
 155
 156 TAILQ_HEAD(port_out_type_tailq, port_out_type);
 157
 158 struct port_out {
 159         TAILQ_ENTRY(port_out) node;
 160         struct port_out_type *type;
 161         void *obj;
 162         uint32_t id;
 163 };
 164
 165 TAILQ_HEAD(port_out_tailq, port_out);
 166
 167 struct port_out_runtime {
 168         rte_swx_port_out_pkt_tx_t pkt_tx;
 169         rte_swx_port_out_flush_t flush;
 170         void *obj;
 171 };
 172
 173 /*
 174  * Extern object.
 175  */
 176 struct extern_type_member_func {
 177         TAILQ_ENTRY(extern_type_member_func) node;
 178         char name[RTE_SWX_NAME_SIZE];
 179         rte_swx_extern_type_member_func_t func;
 180         uint32_t id;
 181 };
 182
 183 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 184
 185 struct extern_type {
 186         TAILQ_ENTRY(extern_type) node;
 187         char name[RTE_SWX_NAME_SIZE];
 188         struct struct_type *mailbox_struct_type;
 189         rte_swx_extern_type_constructor_t constructor;
 190         rte_swx_extern_type_destructor_t destructor;
 191         struct extern_type_member_func_tailq funcs;
 192         uint32_t n_funcs;
 193 };
 194
 195 TAILQ_HEAD(extern_type_tailq, extern_type);
 196
 197 struct extern_obj {
 198         TAILQ_ENTRY(extern_obj) node;
 199         char name[RTE_SWX_NAME_SIZE];
 200         struct extern_type *type;
 201         void *obj;
 202         uint32_t struct_id;
 203         uint32_t id;
 204 };
 205
 206 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 207
 208 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 209 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 210 #endif
 211
 212 struct extern_obj_runtime {
 213         void *obj;
 214         uint8_t *mailbox;
 215         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 216 };
 217
 218 /*
 219  * Extern function.
 220  */
 221 struct extern_func {
 222         TAILQ_ENTRY(extern_func) node;
 223         char name[RTE_SWX_NAME_SIZE];
 224         struct struct_type *mailbox_struct_type;
 225         rte_swx_extern_func_t func;
 226         uint32_t struct_id;
 227         uint32_t id;
 228 };
 229
 230 TAILQ_HEAD(extern_func_tailq, extern_func);
 231
 232 struct extern_func_runtime {
 233         uint8_t *mailbox;
 234         rte_swx_extern_func_t func;
 235 };
 236
 237 /*
 238  * Header.
 239  */
 240 struct header {
 241         TAILQ_ENTRY(header) node;
 242         char name[RTE_SWX_NAME_SIZE];
 243         struct struct_type *st;
 244         uint32_t struct_id;
 245         uint32_t id;
 246 };
 247
 248 TAILQ_HEAD(header_tailq, header);
 249
 250 struct header_runtime {
 251         uint8_t *ptr0;
 252 };
 253
 254 struct header_out_runtime {
 255         uint8_t *ptr0;
 256         uint8_t *ptr;
 257         uint32_t n_bytes;
 258 };
 259
 260 /*
 261  * Instruction.
 262  */
 263
 264 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 265  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 266  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 267  * when transferred to packet meta-data and in NBO when transferred to packet
 268  * headers.
 269  */
 270
 271 /* Notation conventions:
 272  *    -Header field: H = h.header.field (dst/src)
 273  *    -Meta-data field: M = m.field (dst/src)
 274  *    -Extern object mailbox field: E = e.field (dst/src)
 275  *    -Extern function mailbox field: F = f.field (dst/src)
 276  *    -Table action data field: T = t.field (src only)
 277  *    -Immediate value: I = 32-bit unsigned value (src only)
 278  */
 279
 280 enum instruction_type {
 281         /* rx m.port_in */
 282         INSTR_RX,
 283
 284         /* tx port_out
 285          * port_out = MI
 286          */
 287         INSTR_TX,   /* port_out = M */
 288         INSTR_TX_I, /* port_out = I */
 289
 290         /* extract h.header */
 291         INSTR_HDR_EXTRACT,
 292         INSTR_HDR_EXTRACT2,
 293         INSTR_HDR_EXTRACT3,
 294         INSTR_HDR_EXTRACT4,
 295         INSTR_HDR_EXTRACT5,
 296         INSTR_HDR_EXTRACT6,
 297         INSTR_HDR_EXTRACT7,
 298         INSTR_HDR_EXTRACT8,
 299
 300         /* emit h.header */
 301         INSTR_HDR_EMIT,
 302         INSTR_HDR_EMIT_TX,
 303         INSTR_HDR_EMIT2_TX,
 304         INSTR_HDR_EMIT3_TX,
 305         INSTR_HDR_EMIT4_TX,
 306         INSTR_HDR_EMIT5_TX,
 307         INSTR_HDR_EMIT6_TX,
 308         INSTR_HDR_EMIT7_TX,
 309         INSTR_HDR_EMIT8_TX,
 310
 311         /* validate h.header */
 312         INSTR_HDR_VALIDATE,
 313
 314         /* invalidate h.header */
 315         INSTR_HDR_INVALIDATE,
 316
 317         /* mov dst src
 318          * dst = src
 319          * dst = HMEF, src = HMEFTI
 320          */
 321         INSTR_MOV,   /* dst = MEF, src = MEFT */
 322         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 323         INSTR_MOV_I, /* dst = HMEF, src = I */
 324
 325         /* dma h.header t.field
 326          * memcpy(h.header, t.field, sizeof(h.header))
 327          */
 328         INSTR_DMA_HT,
 329         INSTR_DMA_HT2,
 330         INSTR_DMA_HT3,
 331         INSTR_DMA_HT4,
 332         INSTR_DMA_HT5,
 333         INSTR_DMA_HT6,
 334         INSTR_DMA_HT7,
 335         INSTR_DMA_HT8,
 336
 337         /* add dst src
 338          * dst += src
 339          * dst = HMEF, src = HMEFTI
 340          */
 341         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 342         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 343         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 344         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 345         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 346         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 347
 348         /* sub dst src
 349          * dst -= src
 350          * dst = HMEF, src = HMEFTI
 351          */
 352         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 353         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 354         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 355         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 356         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 357         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 358
 359         /* ckadd dst src
 360          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 361          * dst = H, src = {H, h.header}
 362          */
 363         INSTR_ALU_CKADD_FIELD,    /* src = H */
 364         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 365         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 366
 367         /* cksub dst src
 368          * dst = dst '- src
 369          * dst = H, src = H
 370          */
 371         INSTR_ALU_CKSUB_FIELD,
 372
 373         /* and dst src
 374          * dst &= src
 375          * dst = HMEF, src = HMEFTI
 376          */
 377         INSTR_ALU_AND,   /* dst = MEF, src = MEFT */
 378         INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 379         INSTR_ALU_AND_I, /* dst = HMEF, src = I */
 380
 381         /* or dst src
 382          * dst |= src
 383          * dst = HMEF, src = HMEFTI
 384          */
 385         INSTR_ALU_OR,   /* dst = MEF, src = MEFT */
 386         INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 387         INSTR_ALU_OR_I, /* dst = HMEF, src = I */
 388
 389         /* xor dst src
 390          * dst ^= src
 391          * dst = HMEF, src = HMEFTI
 392          */
 393         INSTR_ALU_XOR,   /* dst = MEF, src = MEFT */
 394         INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 395         INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
 396
 397         /* shl dst src
 398          * dst <<= src
 399          * dst = HMEF, src = HMEFTI
 400          */
 401         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 402         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 403         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 404         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 405         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 406         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 407
 408         /* shr dst src
 409          * dst >>= src
 410          * dst = HMEF, src = HMEFTI
 411          */
 412         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 413         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 414         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 415         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 416         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 417         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 418
 419         /* regprefetch REGARRAY index
 420          * prefetch REGARRAY[index]
 421          * index = HMEFTI
 422          */
 423         INSTR_REGPREFETCH_RH, /* index = H */
 424         INSTR_REGPREFETCH_RM, /* index = MEFT */
 425         INSTR_REGPREFETCH_RI, /* index = I */
 426
 427         /* regrd dst REGARRAY index
 428          * dst = REGARRAY[index]
 429          * dst = HMEF, index = HMEFTI
 430          */
 431         INSTR_REGRD_HRH, /* dst = H, index = H */
 432         INSTR_REGRD_HRM, /* dst = H, index = MEFT */
 433         INSTR_REGRD_HRI, /* dst = H, index = I */
 434         INSTR_REGRD_MRH, /* dst = MEF, index = H */
 435         INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
 436         INSTR_REGRD_MRI, /* dst = MEF, index = I */
 437
 438         /* regwr REGARRAY index src
 439          * REGARRAY[index] = src
 440          * index = HMEFTI, src = HMEFTI
 441          */
 442         INSTR_REGWR_RHH, /* index = H, src = H */
 443         INSTR_REGWR_RHM, /* index = H, src = MEFT */
 444         INSTR_REGWR_RHI, /* index = H, src = I */
 445         INSTR_REGWR_RMH, /* index = MEFT, src = H */
 446         INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
 447         INSTR_REGWR_RMI, /* index = MEFT, src = I */
 448         INSTR_REGWR_RIH, /* index = I, src = H */
 449         INSTR_REGWR_RIM, /* index = I, src = MEFT */
 450         INSTR_REGWR_RII, /* index = I, src = I */
 451
 452         /* regadd REGARRAY index src
 453          * REGARRAY[index] += src
 454          * index = HMEFTI, src = HMEFTI
 455          */
 456         INSTR_REGADD_RHH, /* index = H, src = H */
 457         INSTR_REGADD_RHM, /* index = H, src = MEFT */
 458         INSTR_REGADD_RHI, /* index = H, src = I */
 459         INSTR_REGADD_RMH, /* index = MEFT, src = H */
 460         INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
 461         INSTR_REGADD_RMI, /* index = MEFT, src = I */
 462         INSTR_REGADD_RIH, /* index = I, src = H */
 463         INSTR_REGADD_RIM, /* index = I, src = MEFT */
 464         INSTR_REGADD_RII, /* index = I, src = I */
 465
 466         /* metprefetch METARRAY index
 467          * prefetch METARRAY[index]
 468          * index = HMEFTI
 469          */
 470         INSTR_METPREFETCH_H, /* index = H */
 471         INSTR_METPREFETCH_M, /* index = MEFT */
 472         INSTR_METPREFETCH_I, /* index = I */
 473
 474         /* meter METARRAY index length color_in color_out
 475          * color_out = meter(METARRAY[index], length, color_in)
 476          * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
 477          */
 478         INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
 479         INSTR_METER_HHI, /* index = H, length = H, color_in = I */
 480         INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
 481         INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
 482         INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
 483         INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
 484         INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
 485         INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
 486         INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
 487         INSTR_METER_IHI, /* index = I, length = H, color_in = I */
 488         INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
 489         INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
 490
 491         /* table TABLE */
 492         INSTR_TABLE,
 493
 494         /* extern e.obj.func */
 495         INSTR_EXTERN_OBJ,
 496
 497         /* extern f.func */
 498         INSTR_EXTERN_FUNC,
 499
 500         /* jmp LABEL
 501          * Unconditional jump
 502          */
 503         INSTR_JMP,
 504
 505         /* jmpv LABEL h.header
 506          * Jump if header is valid
 507          */
 508         INSTR_JMP_VALID,
 509
 510         /* jmpnv LABEL h.header
 511          * Jump if header is invalid
 512          */
 513         INSTR_JMP_INVALID,
 514
 515         /* jmph LABEL
 516          * Jump if table lookup hit
 517          */
 518         INSTR_JMP_HIT,
 519
 520         /* jmpnh LABEL
 521          * Jump if table lookup miss
 522          */
 523         INSTR_JMP_MISS,
 524
 525         /* jmpa LABEL ACTION
 526          * Jump if action run
 527          */
 528         INSTR_JMP_ACTION_HIT,
 529
 530         /* jmpna LABEL ACTION
 531          * Jump if action not run
 532          */
 533         INSTR_JMP_ACTION_MISS,
 534
 535         /* jmpeq LABEL a b
 536          * Jump is a is equal to b
 537          * a = HMEFT, b = HMEFTI
 538          */
 539         INSTR_JMP_EQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
 540         INSTR_JMP_EQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
 541         INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 542
 543         /* jmpneq LABEL a b
 544          * Jump is a is not equal to b
 545          * a = HMEFT, b = HMEFTI
 546          */
 547         INSTR_JMP_NEQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
 548         INSTR_JMP_NEQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
 549         INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 550
 551         /* jmplt LABEL a b
 552          * Jump if a is less than b
 553          * a = HMEFT, b = HMEFTI
 554          */
 555         INSTR_JMP_LT,    /* a = MEF, b = MEF */
 556         INSTR_JMP_LT_MH, /* a = MEF, b = H */
 557         INSTR_JMP_LT_HM, /* a = H, b = MEF */
 558         INSTR_JMP_LT_HH, /* a = H, b = H */
 559         INSTR_JMP_LT_MI, /* a = MEF, b = I */
 560         INSTR_JMP_LT_HI, /* a = H, b = I */
 561
 562         /* jmpgt LABEL a b
 563          * Jump if a is greater than b
 564          * a = HMEFT, b = HMEFTI
 565          */
 566         INSTR_JMP_GT,    /* a = MEF, b = MEF */
 567         INSTR_JMP_GT_MH, /* a = MEF, b = H */
 568         INSTR_JMP_GT_HM, /* a = H, b = MEF */
 569         INSTR_JMP_GT_HH, /* a = H, b = H */
 570         INSTR_JMP_GT_MI, /* a = MEF, b = I */
 571         INSTR_JMP_GT_HI, /* a = H, b = I */
 572
 573         /* return
 574          * Return from action
 575          */
 576         INSTR_RETURN,
 577 };
 578
 579 struct instr_operand {
 580         uint8_t struct_id;
 581         uint8_t n_bits;
 582         uint8_t offset;
 583         uint8_t pad;
 584 };
 585
 586 struct instr_io {
 587         struct {
 588                 union {
 589                         struct {
 590                                 uint8_t offset;
 591                                 uint8_t n_bits;
 592                                 uint8_t pad[2];
 593                         };
 594
 595                         uint32_t val;
 596                 };
 597         } io;
 598
 599         struct {
 600                 uint8_t header_id[8];
 601                 uint8_t struct_id[8];
 602                 uint8_t n_bytes[8];
 603         } hdr;
 604 };
 605
 606 struct instr_hdr_validity {
 607         uint8_t header_id;
 608 };
 609
 610 struct instr_table {
 611         uint8_t table_id;
 612 };
 613
 614 struct instr_extern_obj {
 615         uint8_t ext_obj_id;
 616         uint8_t func_id;
 617 };
 618
 619 struct instr_extern_func {
 620         uint8_t ext_func_id;
 621 };
 622
 623 struct instr_dst_src {
 624         struct instr_operand dst;
 625         union {
 626                 struct instr_operand src;
 627                 uint64_t src_val;
 628         };
 629 };
 630
 631 struct instr_regarray {
 632         uint8_t regarray_id;
 633         uint8_t pad[3];
 634
 635         union {
 636                 struct instr_operand idx;
 637                 uint32_t idx_val;
 638         };
 639
 640         union {
 641                 struct instr_operand dstsrc;
 642                 uint64_t dstsrc_val;
 643         };
 644 };
 645
 646 struct instr_meter {
 647         uint8_t metarray_id;
 648         uint8_t pad[3];
 649
 650         union {
 651                 struct instr_operand idx;
 652                 uint32_t idx_val;
 653         };
 654
 655         struct instr_operand length;
 656
 657         union {
 658                 struct instr_operand color_in;
 659                 uint32_t color_in_val;
 660         };
 661
 662         struct instr_operand color_out;
 663 };
 664
 665 struct instr_dma {
 666         struct {
 667                 uint8_t header_id[8];
 668                 uint8_t struct_id[8];
 669         } dst;
 670
 671         struct {
 672                 uint8_t offset[8];
 673         } src;
 674
 675         uint16_t n_bytes[8];
 676 };
 677
 678 struct instr_jmp {
 679         struct instruction *ip;
 680
 681         union {
 682                 struct instr_operand a;
 683                 uint8_t header_id;
 684                 uint8_t action_id;
 685         };
 686
 687         union {
 688                 struct instr_operand b;
 689                 uint64_t b_val;
 690         };
 691 };
 692
 693 struct instruction {
 694         enum instruction_type type;
 695         union {
 696                 struct instr_io io;
 697                 struct instr_hdr_validity valid;
 698                 struct instr_dst_src mov;
 699                 struct instr_regarray regarray;
 700                 struct instr_meter meter;
 701                 struct instr_dma dma;
 702                 struct instr_dst_src alu;
 703                 struct instr_table table;
 704                 struct instr_extern_obj ext_obj;
 705                 struct instr_extern_func ext_func;
 706                 struct instr_jmp jmp;
 707         };
 708 };
 709
 710 struct instruction_data {
 711         char label[RTE_SWX_NAME_SIZE];
 712         char jmp_label[RTE_SWX_NAME_SIZE];
 713         uint32_t n_users; /* user = jmp instruction to this instruction. */
 714         int invalid;
 715 };
 716
 717 /*
 718  * Action.
 719  */
 720 struct action {
 721         TAILQ_ENTRY(action) node;
 722         char name[RTE_SWX_NAME_SIZE];
 723         struct struct_type *st;
 724         struct instruction *instructions;
 725         uint32_t n_instructions;
 726         uint32_t id;
 727 };
 728
 729 TAILQ_HEAD(action_tailq, action);
 730
 731 /*
 732  * Table.
 733  */
 734 struct table_type {
 735         TAILQ_ENTRY(table_type) node;
 736         char name[RTE_SWX_NAME_SIZE];
 737         enum rte_swx_table_match_type match_type;
 738         struct rte_swx_table_ops ops;
 739 };
 740
 741 TAILQ_HEAD(table_type_tailq, table_type);
 742
 743 struct match_field {
 744         enum rte_swx_table_match_type match_type;
 745         struct field *field;
 746 };
 747
 748 struct table {
 749         TAILQ_ENTRY(table) node;
 750         char name[RTE_SWX_NAME_SIZE];
 751         char args[RTE_SWX_NAME_SIZE];
 752         struct table_type *type; /* NULL when n_fields == 0. */
 753
 754         /* Match. */
 755         struct match_field *fields;
 756         uint32_t n_fields;
 757         struct header *header; /* Only valid when n_fields > 0. */
 758
 759         /* Action. */
 760         struct action **actions;
 761         struct action *default_action;
 762         uint8_t *default_action_data;
 763         uint32_t n_actions;
 764         int default_action_is_const;
 765         uint32_t action_data_size_max;
 766
 767         uint32_t size;
 768         uint32_t id;
 769 };
 770
 771 TAILQ_HEAD(table_tailq, table);
 772
 773 struct table_runtime {
 774         rte_swx_table_lookup_t func;
 775         void *mailbox;
 776         uint8_t **key;
 777 };
 778
 779 /*
 780  * Register array.
 781  */
 782 struct regarray {
 783         TAILQ_ENTRY(regarray) node;
 784         char name[RTE_SWX_NAME_SIZE];
 785         uint64_t init_val;
 786         uint32_t size;
 787         uint32_t id;
 788 };
 789
 790 TAILQ_HEAD(regarray_tailq, regarray);
 791
 792 struct regarray_runtime {
 793         uint64_t *regarray;
 794         uint32_t size_mask;
 795 };
 796
 797 /*
 798  * Meter array.
 799  */
 800 struct meter_profile {
 801         TAILQ_ENTRY(meter_profile) node;
 802         char name[RTE_SWX_NAME_SIZE];
 803         struct rte_meter_trtcm_params params;
 804         struct rte_meter_trtcm_profile profile;
 805         uint32_t n_users;
 806 };
 807
 808 TAILQ_HEAD(meter_profile_tailq, meter_profile);
 809
 810 struct metarray {
 811         TAILQ_ENTRY(metarray) node;
 812         char name[RTE_SWX_NAME_SIZE];
 813         uint32_t size;
 814         uint32_t id;
 815 };
 816
 817 TAILQ_HEAD(metarray_tailq, metarray);
 818
 819 struct meter {
 820         struct rte_meter_trtcm m;
 821         struct meter_profile *profile;
 822         enum rte_color color_mask;
 823         uint8_t pad[20];
 824
 825         uint64_t n_pkts[RTE_COLORS];
 826         uint64_t n_bytes[RTE_COLORS];
 827 };
 828
 829 struct metarray_runtime {
 830         struct meter *metarray;
 831         uint32_t size_mask;
 832 };
 833
 834 /*
 835  * Pipeline.
 836  */
 837 struct thread {
 838         /* Packet. */
 839         struct rte_swx_pkt pkt;
 840         uint8_t *ptr;
 841
 842         /* Structures. */
 843         uint8_t **structs;
 844
 845         /* Packet headers. */
 846         struct header_runtime *headers; /* Extracted or generated headers. */
 847         struct header_out_runtime *headers_out; /* Emitted headers. */
 848         uint8_t *header_storage;
 849         uint8_t *header_out_storage;
 850         uint64_t valid_headers;
 851         uint32_t n_headers_out;
 852
 853         /* Packet meta-data. */
 854         uint8_t *metadata;
 855
 856         /* Tables. */
 857         struct table_runtime *tables;
 858         struct rte_swx_table_state *table_state;
 859         uint64_t action_id;
 860         int hit; /* 0 = Miss, 1 = Hit. */
 861
 862         /* Extern objects and functions. */
 863         struct extern_obj_runtime *extern_objs;
 864         struct extern_func_runtime *extern_funcs;
 865
 866         /* Instructions. */
 867         struct instruction *ip;
 868         struct instruction *ret;
 869 };
 870
 871 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 872 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 873 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 874
 875 #define HEADER_VALID(thread, header_id) \
 876         MASK64_BIT_GET((thread)->valid_headers, header_id)
 877
 878 #define ALU(thread, ip, operator)  \
 879 {                                                                              \
 880         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 881         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 882         uint64_t dst64 = *dst64_ptr;                                           \
 883         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 884         uint64_t dst = dst64 & dst64_mask;                                     \
 885                                                                                \
 886         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 887         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 888         uint64_t src64 = *src64_ptr;                                           \
 889         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 890         uint64_t src = src64 & src64_mask;                                     \
 891                                                                                \
 892         uint64_t result = dst operator src;                                    \
 893                                                                                \
 894         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 895 }
 896
 897 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 898
 899 #define ALU_S(thread, ip, operator)  \
 900 {                                                                              \
 901         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 902         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 903         uint64_t dst64 = *dst64_ptr;                                           \
 904         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 905         uint64_t dst = dst64 & dst64_mask;                                     \
 906                                                                                \
 907         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 908         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 909         uint64_t src64 = *src64_ptr;                                           \
 910         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 911                                                                                \
 912         uint64_t result = dst operator src;                                    \
 913                                                                                \
 914         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 915 }
 916
 917 #define ALU_MH ALU_S
 918
 919 #define ALU_HM(thread, ip, operator)  \
 920 {                                                                              \
 921         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 922         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 923         uint64_t dst64 = *dst64_ptr;                                           \
 924         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 925         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 926                                                                                \
 927         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 928         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 929         uint64_t src64 = *src64_ptr;                                           \
 930         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 931         uint64_t src = src64 & src64_mask;                                     \
 932                                                                                \
 933         uint64_t result = dst operator src;                                    \
 934         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 935                                                                                \
 936         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 937 }
 938
 939 #define ALU_HH(thread, ip, operator)  \
 940 {                                                                              \
 941         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 942         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 943         uint64_t dst64 = *dst64_ptr;                                           \
 944         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 945         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 946                                                                                \
 947         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 948         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 949         uint64_t src64 = *src64_ptr;                                           \
 950         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 951                                                                                \
 952         uint64_t result = dst operator src;                                    \
 953         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 954                                                                                \
 955         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 956 }
 957
 958 #else
 959
 960 #define ALU_S ALU
 961 #define ALU_MH ALU
 962 #define ALU_HM ALU
 963 #define ALU_HH ALU
 964
 965 #endif
 966
 967 #define ALU_I(thread, ip, operator)  \
 968 {                                                                              \
 969         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 970         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 971         uint64_t dst64 = *dst64_ptr;                                           \
 972         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 973         uint64_t dst = dst64 & dst64_mask;                                     \
 974                                                                                \
 975         uint64_t src = (ip)->alu.src_val;                                      \
 976                                                                                \
 977         uint64_t result = dst operator src;                                    \
 978                                                                                \
 979         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 980 }
 981
 982 #define ALU_MI ALU_I
 983
 984 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 985
 986 #define ALU_HI(thread, ip, operator)  \
 987 {                                                                              \
 988         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 989         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 990         uint64_t dst64 = *dst64_ptr;                                           \
 991         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 992         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 993                                                                                \
 994         uint64_t src = (ip)->alu.src_val;                                      \
 995                                                                                \
 996         uint64_t result = dst operator src;                                    \
 997         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 998                                                                                \
 999         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1000 }
1001
1002 #else
1003
1004 #define ALU_HI ALU_I
1005
1006 #endif
1007
1008 #define MOV(thread, ip)  \
1009 {                                                                              \
1010         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1011         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1012         uint64_t dst64 = *dst64_ptr;                                           \
1013         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1014                                                                                \
1015         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1016         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1017         uint64_t src64 = *src64_ptr;                                           \
1018         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1019         uint64_t src = src64 & src64_mask;                                     \
1020                                                                                \
1021         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1022 }
1023
1024 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1025
1026 #define MOV_S(thread, ip)  \
1027 {                                                                              \
1028         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1029         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1030         uint64_t dst64 = *dst64_ptr;                                           \
1031         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1032                                                                                \
1033         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1034         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1035         uint64_t src64 = *src64_ptr;                                           \
1036         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
1037                                                                                \
1038         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1039 }
1040
1041 #else
1042
1043 #define MOV_S MOV
1044
1045 #endif
1046
1047 #define MOV_I(thread, ip)  \
1048 {                                                                              \
1049         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1050         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1051         uint64_t dst64 = *dst64_ptr;                                           \
1052         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1053                                                                                \
1054         uint64_t src = (ip)->mov.src_val;                                      \
1055                                                                                \
1056         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1057 }
1058
1059 #define JMP_CMP(thread, ip, operator)  \
1060 {                                                                              \
1061         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1062         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1063         uint64_t a64 = *a64_ptr;                                               \
1064         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1065         uint64_t a = a64 & a64_mask;                                           \
1066                                                                                \
1067         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1068         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1069         uint64_t b64 = *b64_ptr;                                               \
1070         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1071         uint64_t b = b64 & b64_mask;                                           \
1072                                                                                \
1073         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1074 }
1075
1076 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1077
1078 #define JMP_CMP_S(thread, ip, operator)  \
1079 {                                                                              \
1080         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1081         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1082         uint64_t a64 = *a64_ptr;                                               \
1083         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1084         uint64_t a = a64 & a64_mask;                                           \
1085                                                                                \
1086         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1087         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1088         uint64_t b64 = *b64_ptr;                                               \
1089         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1090                                                                                \
1091         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1092 }
1093
1094 #define JMP_CMP_MH JMP_CMP_S
1095
1096 #define JMP_CMP_HM(thread, ip, operator)  \
1097 {                                                                              \
1098         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1099         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1100         uint64_t a64 = *a64_ptr;                                               \
1101         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1102                                                                                \
1103         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1104         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1105         uint64_t b64 = *b64_ptr;                                               \
1106         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1107         uint64_t b = b64 & b64_mask;                                           \
1108                                                                                \
1109         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1110 }
1111
1112 #define JMP_CMP_HH(thread, ip, operator)  \
1113 {                                                                              \
1114         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1115         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1116         uint64_t a64 = *a64_ptr;                                               \
1117         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1118                                                                                \
1119         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1120         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1121         uint64_t b64 = *b64_ptr;                                               \
1122         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1123                                                                                \
1124         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1125 }
1126
1127 #else
1128
1129 #define JMP_CMP_S JMP_CMP
1130 #define JMP_CMP_MH JMP_CMP
1131 #define JMP_CMP_HM JMP_CMP
1132 #define JMP_CMP_HH JMP_CMP
1133
1134 #endif
1135
1136 #define JMP_CMP_I(thread, ip, operator)  \
1137 {                                                                              \
1138         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1139         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1140         uint64_t a64 = *a64_ptr;                                               \
1141         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1142         uint64_t a = a64 & a64_mask;                                           \
1143                                                                                \
1144         uint64_t b = (ip)->jmp.b_val;                                          \
1145                                                                                \
1146         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1147 }
1148
1149 #define JMP_CMP_MI JMP_CMP_I
1150
1151 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1152
1153 #define JMP_CMP_HI(thread, ip, operator)  \
1154 {                                                                              \
1155         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1156         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1157         uint64_t a64 = *a64_ptr;                                               \
1158         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1159                                                                                \
1160         uint64_t b = (ip)->jmp.b_val;                                          \
1161                                                                                \
1162         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1163 }
1164
1165 #else
1166
1167 #define JMP_CMP_HI JMP_CMP_I
1168
1169 #endif
1170
1171 #define METADATA_READ(thread, offset, n_bits)                                  \
1172 ({                                                                             \
1173         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1174         uint64_t m64 = *m64_ptr;                                               \
1175         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1176         (m64 & m64_mask);                                                      \
1177 })
1178
1179 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
1180 {                                                                              \
1181         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1182         uint64_t m64 = *m64_ptr;                                               \
1183         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1184                                                                                \
1185         uint64_t m_new = value;                                                \
1186                                                                                \
1187         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
1188 }
1189
1190 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1191 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1192 #endif
1193
1194 struct rte_swx_pipeline {
1195         struct struct_type_tailq struct_types;
1196         struct port_in_type_tailq port_in_types;
1197         struct port_in_tailq ports_in;
1198         struct port_out_type_tailq port_out_types;
1199         struct port_out_tailq ports_out;
1200         struct extern_type_tailq extern_types;
1201         struct extern_obj_tailq extern_objs;
1202         struct extern_func_tailq extern_funcs;
1203         struct header_tailq headers;
1204         struct struct_type *metadata_st;
1205         uint32_t metadata_struct_id;
1206         struct action_tailq actions;
1207         struct table_type_tailq table_types;
1208         struct table_tailq tables;
1209         struct regarray_tailq regarrays;
1210         struct meter_profile_tailq meter_profiles;
1211         struct metarray_tailq metarrays;
1212
1213         struct port_in_runtime *in;
1214         struct port_out_runtime *out;
1215         struct instruction **action_instructions;
1216         struct rte_swx_table_state *table_state;
1217         struct regarray_runtime *regarray_runtime;
1218         struct metarray_runtime *metarray_runtime;
1219         struct instruction *instructions;
1220         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1221
1222         uint32_t n_structs;
1223         uint32_t n_ports_in;
1224         uint32_t n_ports_out;
1225         uint32_t n_extern_objs;
1226         uint32_t n_extern_funcs;
1227         uint32_t n_actions;
1228         uint32_t n_tables;
1229         uint32_t n_regarrays;
1230         uint32_t n_metarrays;
1231         uint32_t n_headers;
1232         uint32_t thread_id;
1233         uint32_t port_id;
1234         uint32_t n_instructions;
1235         int build_done;
1236         int numa_node;
1237 };
1238
1239 /*
1240  * Struct.
1241  */
1242 static struct struct_type *
1243 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1244 {
1245         struct struct_type *elem;
1246
1247         TAILQ_FOREACH(elem, &p->struct_types, node)
1248                 if (strcmp(elem->name, name) == 0)
1249                         return elem;
1250
1251         return NULL;
1252 }
1253
1254 static struct field *
1255 struct_type_field_find(struct struct_type *st, const char *name)
1256 {
1257         uint32_t i;
1258
1259         for (i = 0; i < st->n_fields; i++) {
1260                 struct field *f = &st->fields[i];
1261
1262                 if (strcmp(f->name, name) == 0)
1263                         return f;
1264         }
1265
1266         return NULL;
1267 }
1268
1269 int
1270 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1271                                       const char *name,
1272                                       struct rte_swx_field_params *fields,
1273                                       uint32_t n_fields)
1274 {
1275         struct struct_type *st;
1276         uint32_t i;
1277
1278         CHECK(p, EINVAL);
1279         CHECK_NAME(name, EINVAL);
1280         CHECK(fields, EINVAL);
1281         CHECK(n_fields, EINVAL);
1282
1283         for (i = 0; i < n_fields; i++) {
1284                 struct rte_swx_field_params *f = &fields[i];
1285                 uint32_t j;
1286
1287                 CHECK_NAME(f->name, EINVAL);
1288                 CHECK(f->n_bits, EINVAL);
1289                 CHECK(f->n_bits <= 64, EINVAL);
1290                 CHECK((f->n_bits & 7) == 0, EINVAL);
1291
1292                 for (j = 0; j < i; j++) {
1293                         struct rte_swx_field_params *f_prev = &fields[j];
1294
1295                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
1296                 }
1297         }
1298
1299         CHECK(!struct_type_find(p, name), EEXIST);
1300
1301         /* Node allocation. */
1302         st = calloc(1, sizeof(struct struct_type));
1303         CHECK(st, ENOMEM);
1304
1305         st->fields = calloc(n_fields, sizeof(struct field));
1306         if (!st->fields) {
1307                 free(st);
1308                 CHECK(0, ENOMEM);
1309         }
1310
1311         /* Node initialization. */
1312         strcpy(st->name, name);
1313         for (i = 0; i < n_fields; i++) {
1314                 struct field *dst = &st->fields[i];
1315                 struct rte_swx_field_params *src = &fields[i];
1316
1317                 strcpy(dst->name, src->name);
1318                 dst->n_bits = src->n_bits;
1319                 dst->offset = st->n_bits;
1320
1321                 st->n_bits += src->n_bits;
1322         }
1323         st->n_fields = n_fields;
1324
1325         /* Node add to tailq. */
1326         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1327
1328         return 0;
1329 }
1330
1331 static int
1332 struct_build(struct rte_swx_pipeline *p)
1333 {
1334         uint32_t i;
1335
1336         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1337                 struct thread *t = &p->threads[i];
1338
1339                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1340                 CHECK(t->structs, ENOMEM);
1341         }
1342
1343         return 0;
1344 }
1345
1346 static void
1347 struct_build_free(struct rte_swx_pipeline *p)
1348 {
1349         uint32_t i;
1350
1351         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1352                 struct thread *t = &p->threads[i];
1353
1354                 free(t->structs);
1355                 t->structs = NULL;
1356         }
1357 }
1358
1359 static void
1360 struct_free(struct rte_swx_pipeline *p)
1361 {
1362         struct_build_free(p);
1363
1364         /* Struct types. */
1365         for ( ; ; ) {
1366                 struct struct_type *elem;
1367
1368                 elem = TAILQ_FIRST(&p->struct_types);
1369                 if (!elem)
1370                         break;
1371
1372                 TAILQ_REMOVE(&p->struct_types, elem, node);
1373                 free(elem->fields);
1374                 free(elem);
1375         }
1376 }
1377
1378 /*
1379  * Input port.
1380  */
1381 static struct port_in_type *
1382 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1383 {
1384         struct port_in_type *elem;
1385
1386         if (!name)
1387                 return NULL;
1388
1389         TAILQ_FOREACH(elem, &p->port_in_types, node)
1390                 if (strcmp(elem->name, name) == 0)
1391                         return elem;
1392
1393         return NULL;
1394 }
1395
1396 int
1397 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1398                                        const char *name,
1399                                        struct rte_swx_port_in_ops *ops)
1400 {
1401         struct port_in_type *elem;
1402
1403         CHECK(p, EINVAL);
1404         CHECK_NAME(name, EINVAL);
1405         CHECK(ops, EINVAL);
1406         CHECK(ops->create, EINVAL);
1407         CHECK(ops->free, EINVAL);
1408         CHECK(ops->pkt_rx, EINVAL);
1409         CHECK(ops->stats_read, EINVAL);
1410
1411         CHECK(!port_in_type_find(p, name), EEXIST);
1412
1413         /* Node allocation. */
1414         elem = calloc(1, sizeof(struct port_in_type));
1415         CHECK(elem, ENOMEM);
1416
1417         /* Node initialization. */
1418         strcpy(elem->name, name);
1419         memcpy(&elem->ops, ops, sizeof(*ops));
1420
1421         /* Node add to tailq. */
1422         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1423
1424         return 0;
1425 }
1426
1427 static struct port_in *
1428 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1429 {
1430         struct port_in *port;
1431
1432         TAILQ_FOREACH(port, &p->ports_in, node)
1433                 if (port->id == port_id)
1434                         return port;
1435
1436         return NULL;
1437 }
1438
1439 int
1440 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1441                                 uint32_t port_id,
1442                                 const char *port_type_name,
1443                                 void *args)
1444 {
1445         struct port_in_type *type = NULL;
1446         struct port_in *port = NULL;
1447         void *obj = NULL;
1448
1449         CHECK(p, EINVAL);
1450
1451         CHECK(!port_in_find(p, port_id), EINVAL);
1452
1453         CHECK_NAME(port_type_name, EINVAL);
1454         type = port_in_type_find(p, port_type_name);
1455         CHECK(type, EINVAL);
1456
1457         obj = type->ops.create(args);
1458         CHECK(obj, ENODEV);
1459
1460         /* Node allocation. */
1461         port = calloc(1, sizeof(struct port_in));
1462         CHECK(port, ENOMEM);
1463
1464         /* Node initialization. */
1465         port->type = type;
1466         port->obj = obj;
1467         port->id = port_id;
1468
1469         /* Node add to tailq. */
1470         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1471         if (p->n_ports_in < port_id + 1)
1472                 p->n_ports_in = port_id + 1;
1473
1474         return 0;
1475 }
1476
1477 static int
1478 port_in_build(struct rte_swx_pipeline *p)
1479 {
1480         struct port_in *port;
1481         uint32_t i;
1482
1483         CHECK(p->n_ports_in, EINVAL);
1484         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1485
1486         for (i = 0; i < p->n_ports_in; i++)
1487                 CHECK(port_in_find(p, i), EINVAL);
1488
1489         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1490         CHECK(p->in, ENOMEM);
1491
1492         TAILQ_FOREACH(port, &p->ports_in, node) {
1493                 struct port_in_runtime *in = &p->in[port->id];
1494
1495                 in->pkt_rx = port->type->ops.pkt_rx;
1496                 in->obj = port->obj;
1497         }
1498
1499         return 0;
1500 }
1501
1502 static void
1503 port_in_build_free(struct rte_swx_pipeline *p)
1504 {
1505         free(p->in);
1506         p->in = NULL;
1507 }
1508
1509 static void
1510 port_in_free(struct rte_swx_pipeline *p)
1511 {
1512         port_in_build_free(p);
1513
1514         /* Input ports. */
1515         for ( ; ; ) {
1516                 struct port_in *port;
1517
1518                 port = TAILQ_FIRST(&p->ports_in);
1519                 if (!port)
1520                         break;
1521
1522                 TAILQ_REMOVE(&p->ports_in, port, node);
1523                 port->type->ops.free(port->obj);
1524                 free(port);
1525         }
1526
1527         /* Input port types. */
1528         for ( ; ; ) {
1529                 struct port_in_type *elem;
1530
1531                 elem = TAILQ_FIRST(&p->port_in_types);
1532                 if (!elem)
1533                         break;
1534
1535                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1536                 free(elem);
1537         }
1538 }
1539
1540 /*
1541  * Output port.
1542  */
1543 static struct port_out_type *
1544 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1545 {
1546         struct port_out_type *elem;
1547
1548         if (!name)
1549                 return NULL;
1550
1551         TAILQ_FOREACH(elem, &p->port_out_types, node)
1552                 if (!strcmp(elem->name, name))
1553                         return elem;
1554
1555         return NULL;
1556 }
1557
1558 int
1559 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1560                                         const char *name,
1561                                         struct rte_swx_port_out_ops *ops)
1562 {
1563         struct port_out_type *elem;
1564
1565         CHECK(p, EINVAL);
1566         CHECK_NAME(name, EINVAL);
1567         CHECK(ops, EINVAL);
1568         CHECK(ops->create, EINVAL);
1569         CHECK(ops->free, EINVAL);
1570         CHECK(ops->pkt_tx, EINVAL);
1571         CHECK(ops->stats_read, EINVAL);
1572
1573         CHECK(!port_out_type_find(p, name), EEXIST);
1574
1575         /* Node allocation. */
1576         elem = calloc(1, sizeof(struct port_out_type));
1577         CHECK(elem, ENOMEM);
1578
1579         /* Node initialization. */
1580         strcpy(elem->name, name);
1581         memcpy(&elem->ops, ops, sizeof(*ops));
1582
1583         /* Node add to tailq. */
1584         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1585
1586         return 0;
1587 }
1588
1589 static struct port_out *
1590 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1591 {
1592         struct port_out *port;
1593
1594         TAILQ_FOREACH(port, &p->ports_out, node)
1595                 if (port->id == port_id)
1596                         return port;
1597
1598         return NULL;
1599 }
1600
1601 int
1602 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1603                                  uint32_t port_id,
1604                                  const char *port_type_name,
1605                                  void *args)
1606 {
1607         struct port_out_type *type = NULL;
1608         struct port_out *port = NULL;
1609         void *obj = NULL;
1610
1611         CHECK(p, EINVAL);
1612
1613         CHECK(!port_out_find(p, port_id), EINVAL);
1614
1615         CHECK_NAME(port_type_name, EINVAL);
1616         type = port_out_type_find(p, port_type_name);
1617         CHECK(type, EINVAL);
1618
1619         obj = type->ops.create(args);
1620         CHECK(obj, ENODEV);
1621
1622         /* Node allocation. */
1623         port = calloc(1, sizeof(struct port_out));
1624         CHECK(port, ENOMEM);
1625
1626         /* Node initialization. */
1627         port->type = type;
1628         port->obj = obj;
1629         port->id = port_id;
1630
1631         /* Node add to tailq. */
1632         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1633         if (p->n_ports_out < port_id + 1)
1634                 p->n_ports_out = port_id + 1;
1635
1636         return 0;
1637 }
1638
1639 static int
1640 port_out_build(struct rte_swx_pipeline *p)
1641 {
1642         struct port_out *port;
1643         uint32_t i;
1644
1645         CHECK(p->n_ports_out, EINVAL);
1646
1647         for (i = 0; i < p->n_ports_out; i++)
1648                 CHECK(port_out_find(p, i), EINVAL);
1649
1650         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1651         CHECK(p->out, ENOMEM);
1652
1653         TAILQ_FOREACH(port, &p->ports_out, node) {
1654                 struct port_out_runtime *out = &p->out[port->id];
1655
1656                 out->pkt_tx = port->type->ops.pkt_tx;
1657                 out->flush = port->type->ops.flush;
1658                 out->obj = port->obj;
1659         }
1660
1661         return 0;
1662 }
1663
1664 static void
1665 port_out_build_free(struct rte_swx_pipeline *p)
1666 {
1667         free(p->out);
1668         p->out = NULL;
1669 }
1670
1671 static void
1672 port_out_free(struct rte_swx_pipeline *p)
1673 {
1674         port_out_build_free(p);
1675
1676         /* Output ports. */
1677         for ( ; ; ) {
1678                 struct port_out *port;
1679
1680                 port = TAILQ_FIRST(&p->ports_out);
1681                 if (!port)
1682                         break;
1683
1684                 TAILQ_REMOVE(&p->ports_out, port, node);
1685                 port->type->ops.free(port->obj);
1686                 free(port);
1687         }
1688
1689         /* Output port types. */
1690         for ( ; ; ) {
1691                 struct port_out_type *elem;
1692
1693                 elem = TAILQ_FIRST(&p->port_out_types);
1694                 if (!elem)
1695                         break;
1696
1697                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1698                 free(elem);
1699         }
1700 }
1701
1702 /*
1703  * Extern object.
1704  */
1705 static struct extern_type *
1706 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1707 {
1708         struct extern_type *elem;
1709
1710         TAILQ_FOREACH(elem, &p->extern_types, node)
1711                 if (strcmp(elem->name, name) == 0)
1712                         return elem;
1713
1714         return NULL;
1715 }
1716
1717 static struct extern_type_member_func *
1718 extern_type_member_func_find(struct extern_type *type, const char *name)
1719 {
1720         struct extern_type_member_func *elem;
1721
1722         TAILQ_FOREACH(elem, &type->funcs, node)
1723                 if (strcmp(elem->name, name) == 0)
1724                         return elem;
1725
1726         return NULL;
1727 }
1728
1729 static struct extern_obj *
1730 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1731 {
1732         struct extern_obj *elem;
1733
1734         TAILQ_FOREACH(elem, &p->extern_objs, node)
1735                 if (strcmp(elem->name, name) == 0)
1736                         return elem;
1737
1738         return NULL;
1739 }
1740
1741 static struct extern_type_member_func *
1742 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1743                              const char *name,
1744                              struct extern_obj **obj)
1745 {
1746         struct extern_obj *object;
1747         struct extern_type_member_func *func;
1748         char *object_name, *func_name;
1749
1750         if (name[0] != 'e' || name[1] != '.')
1751                 return NULL;
1752
1753         object_name = strdup(&name[2]);
1754         if (!object_name)
1755                 return NULL;
1756
1757         func_name = strchr(object_name, '.');
1758         if (!func_name) {
1759                 free(object_name);
1760                 return NULL;
1761         }
1762
1763         *func_name = 0;
1764         func_name++;
1765
1766         object = extern_obj_find(p, object_name);
1767         if (!object) {
1768                 free(object_name);
1769                 return NULL;
1770         }
1771
1772         func = extern_type_member_func_find(object->type, func_name);
1773         if (!func) {
1774                 free(object_name);
1775                 return NULL;
1776         }
1777
1778         if (obj)
1779                 *obj = object;
1780
1781         free(object_name);
1782         return func;
1783 }
1784
1785 static struct field *
1786 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1787                                const char *name,
1788                                struct extern_obj **object)
1789 {
1790         struct extern_obj *obj;
1791         struct field *f;
1792         char *obj_name, *field_name;
1793
1794         if ((name[0] != 'e') || (name[1] != '.'))
1795                 return NULL;
1796
1797         obj_name = strdup(&name[2]);
1798         if (!obj_name)
1799                 return NULL;
1800
1801         field_name = strchr(obj_name, '.');
1802         if (!field_name) {
1803                 free(obj_name);
1804                 return NULL;
1805         }
1806
1807         *field_name = 0;
1808         field_name++;
1809
1810         obj = extern_obj_find(p, obj_name);
1811         if (!obj) {
1812                 free(obj_name);
1813                 return NULL;
1814         }
1815
1816         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1817         if (!f) {
1818                 free(obj_name);
1819                 return NULL;
1820         }
1821
1822         if (object)
1823                 *object = obj;
1824
1825         free(obj_name);
1826         return f;
1827 }
1828
1829 int
1830 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1831         const char *name,
1832         const char *mailbox_struct_type_name,
1833         rte_swx_extern_type_constructor_t constructor,
1834         rte_swx_extern_type_destructor_t destructor)
1835 {
1836         struct extern_type *elem;
1837         struct struct_type *mailbox_struct_type;
1838
1839         CHECK(p, EINVAL);
1840
1841         CHECK_NAME(name, EINVAL);
1842         CHECK(!extern_type_find(p, name), EEXIST);
1843
1844         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1845         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1846         CHECK(mailbox_struct_type, EINVAL);
1847
1848         CHECK(constructor, EINVAL);
1849         CHECK(destructor, EINVAL);
1850
1851         /* Node allocation. */
1852         elem = calloc(1, sizeof(struct extern_type));
1853         CHECK(elem, ENOMEM);
1854
1855         /* Node initialization. */
1856         strcpy(elem->name, name);
1857         elem->mailbox_struct_type = mailbox_struct_type;
1858         elem->constructor = constructor;
1859         elem->destructor = destructor;
1860         TAILQ_INIT(&elem->funcs);
1861
1862         /* Node add to tailq. */
1863         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1864
1865         return 0;
1866 }
1867
1868 int
1869 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1870         const char *extern_type_name,
1871         const char *name,
1872         rte_swx_extern_type_member_func_t member_func)
1873 {
1874         struct extern_type *type;
1875         struct extern_type_member_func *type_member;
1876
1877         CHECK(p, EINVAL);
1878
1879         CHECK_NAME(extern_type_name, EINVAL);
1880         type = extern_type_find(p, extern_type_name);
1881         CHECK(type, EINVAL);
1882         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1883
1884         CHECK_NAME(name, EINVAL);
1885         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1886
1887         CHECK(member_func, EINVAL);
1888
1889         /* Node allocation. */
1890         type_member = calloc(1, sizeof(struct extern_type_member_func));
1891         CHECK(type_member, ENOMEM);
1892
1893         /* Node initialization. */
1894         strcpy(type_member->name, name);
1895         type_member->func = member_func;
1896         type_member->id = type->n_funcs;
1897
1898         /* Node add to tailq. */
1899         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1900         type->n_funcs++;
1901
1902         return 0;
1903 }
1904
1905 int
1906 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1907                                       const char *extern_type_name,
1908                                       const char *name,
1909                                       const char *args)
1910 {
1911         struct extern_type *type;
1912         struct extern_obj *obj;
1913         void *obj_handle;
1914
1915         CHECK(p, EINVAL);
1916
1917         CHECK_NAME(extern_type_name, EINVAL);
1918         type = extern_type_find(p, extern_type_name);
1919         CHECK(type, EINVAL);
1920
1921         CHECK_NAME(name, EINVAL);
1922         CHECK(!extern_obj_find(p, name), EEXIST);
1923
1924         /* Node allocation. */
1925         obj = calloc(1, sizeof(struct extern_obj));
1926         CHECK(obj, ENOMEM);
1927
1928         /* Object construction. */
1929         obj_handle = type->constructor(args);
1930         if (!obj_handle) {
1931                 free(obj);
1932                 CHECK(0, ENODEV);
1933         }
1934
1935         /* Node initialization. */
1936         strcpy(obj->name, name);
1937         obj->type = type;
1938         obj->obj = obj_handle;
1939         obj->struct_id = p->n_structs;
1940         obj->id = p->n_extern_objs;
1941
1942         /* Node add to tailq. */
1943         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1944         p->n_extern_objs++;
1945         p->n_structs++;
1946
1947         return 0;
1948 }
1949
1950 static int
1951 extern_obj_build(struct rte_swx_pipeline *p)
1952 {
1953         uint32_t i;
1954
1955         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1956                 struct thread *t = &p->threads[i];
1957                 struct extern_obj *obj;
1958
1959                 t->extern_objs = calloc(p->n_extern_objs,
1960                                         sizeof(struct extern_obj_runtime));
1961                 CHECK(t->extern_objs, ENOMEM);
1962
1963                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1964                         struct extern_obj_runtime *r =
1965                                 &t->extern_objs[obj->id];
1966                         struct extern_type_member_func *func;
1967                         uint32_t mailbox_size =
1968                                 obj->type->mailbox_struct_type->n_bits / 8;
1969
1970                         r->obj = obj->obj;
1971
1972                         r->mailbox = calloc(1, mailbox_size);
1973                         CHECK(r->mailbox, ENOMEM);
1974
1975                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1976                                 r->funcs[func->id] = func->func;
1977
1978                         t->structs[obj->struct_id] = r->mailbox;
1979                 }
1980         }
1981
1982         return 0;
1983 }
1984
1985 static void
1986 extern_obj_build_free(struct rte_swx_pipeline *p)
1987 {
1988         uint32_t i;
1989
1990         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1991                 struct thread *t = &p->threads[i];
1992                 uint32_t j;
1993
1994                 if (!t->extern_objs)
1995                         continue;
1996
1997                 for (j = 0; j < p->n_extern_objs; j++) {
1998                         struct extern_obj_runtime *r = &t->extern_objs[j];
1999
2000                         free(r->mailbox);
2001                 }
2002
2003                 free(t->extern_objs);
2004                 t->extern_objs = NULL;
2005         }
2006 }
2007
2008 static void
2009 extern_obj_free(struct rte_swx_pipeline *p)
2010 {
2011         extern_obj_build_free(p);
2012
2013         /* Extern objects. */
2014         for ( ; ; ) {
2015                 struct extern_obj *elem;
2016
2017                 elem = TAILQ_FIRST(&p->extern_objs);
2018                 if (!elem)
2019                         break;
2020
2021                 TAILQ_REMOVE(&p->extern_objs, elem, node);
2022                 if (elem->obj)
2023                         elem->type->destructor(elem->obj);
2024                 free(elem);
2025         }
2026
2027         /* Extern types. */
2028         for ( ; ; ) {
2029                 struct extern_type *elem;
2030
2031                 elem = TAILQ_FIRST(&p->extern_types);
2032                 if (!elem)
2033                         break;
2034
2035                 TAILQ_REMOVE(&p->extern_types, elem, node);
2036
2037                 for ( ; ; ) {
2038                         struct extern_type_member_func *func;
2039
2040                         func = TAILQ_FIRST(&elem->funcs);
2041                         if (!func)
2042                                 break;
2043
2044                         TAILQ_REMOVE(&elem->funcs, func, node);
2045                         free(func);
2046                 }
2047
2048                 free(elem);
2049         }
2050 }
2051
2052 /*
2053  * Extern function.
2054  */
2055 static struct extern_func *
2056 extern_func_find(struct rte_swx_pipeline *p, const char *name)
2057 {
2058         struct extern_func *elem;
2059
2060         TAILQ_FOREACH(elem, &p->extern_funcs, node)
2061                 if (strcmp(elem->name, name) == 0)
2062                         return elem;
2063
2064         return NULL;
2065 }
2066
2067 static struct extern_func *
2068 extern_func_parse(struct rte_swx_pipeline *p,
2069                   const char *name)
2070 {
2071         if (name[0] != 'f' || name[1] != '.')
2072                 return NULL;
2073
2074         return extern_func_find(p, &name[2]);
2075 }
2076
2077 static struct field *
2078 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
2079                                 const char *name,
2080                                 struct extern_func **function)
2081 {
2082         struct extern_func *func;
2083         struct field *f;
2084         char *func_name, *field_name;
2085
2086         if ((name[0] != 'f') || (name[1] != '.'))
2087                 return NULL;
2088
2089         func_name = strdup(&name[2]);
2090         if (!func_name)
2091                 return NULL;
2092
2093         field_name = strchr(func_name, '.');
2094         if (!field_name) {
2095                 free(func_name);
2096                 return NULL;
2097         }
2098
2099         *field_name = 0;
2100         field_name++;
2101
2102         func = extern_func_find(p, func_name);
2103         if (!func) {
2104                 free(func_name);
2105                 return NULL;
2106         }
2107
2108         f = struct_type_field_find(func->mailbox_struct_type, field_name);
2109         if (!f) {
2110                 free(func_name);
2111                 return NULL;
2112         }
2113
2114         if (function)
2115                 *function = func;
2116
2117         free(func_name);
2118         return f;
2119 }
2120
2121 int
2122 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
2123                                       const char *name,
2124                                       const char *mailbox_struct_type_name,
2125                                       rte_swx_extern_func_t func)
2126 {
2127         struct extern_func *f;
2128         struct struct_type *mailbox_struct_type;
2129
2130         CHECK(p, EINVAL);
2131
2132         CHECK_NAME(name, EINVAL);
2133         CHECK(!extern_func_find(p, name), EEXIST);
2134
2135         CHECK_NAME(mailbox_struct_type_name, EINVAL);
2136         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2137         CHECK(mailbox_struct_type, EINVAL);
2138
2139         CHECK(func, EINVAL);
2140
2141         /* Node allocation. */
2142         f = calloc(1, sizeof(struct extern_func));
2143         CHECK(func, ENOMEM);
2144
2145         /* Node initialization. */
2146         strcpy(f->name, name);
2147         f->mailbox_struct_type = mailbox_struct_type;
2148         f->func = func;
2149         f->struct_id = p->n_structs;
2150         f->id = p->n_extern_funcs;
2151
2152         /* Node add to tailq. */
2153         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
2154         p->n_extern_funcs++;
2155         p->n_structs++;
2156
2157         return 0;
2158 }
2159
2160 static int
2161 extern_func_build(struct rte_swx_pipeline *p)
2162 {
2163         uint32_t i;
2164
2165         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2166                 struct thread *t = &p->threads[i];
2167                 struct extern_func *func;
2168
2169                 /* Memory allocation. */
2170                 t->extern_funcs = calloc(p->n_extern_funcs,
2171                                          sizeof(struct extern_func_runtime));
2172                 CHECK(t->extern_funcs, ENOMEM);
2173
2174                 /* Extern function. */
2175                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
2176                         struct extern_func_runtime *r =
2177                                 &t->extern_funcs[func->id];
2178                         uint32_t mailbox_size =
2179                                 func->mailbox_struct_type->n_bits / 8;
2180
2181                         r->func = func->func;
2182
2183                         r->mailbox = calloc(1, mailbox_size);
2184                         CHECK(r->mailbox, ENOMEM);
2185
2186                         t->structs[func->struct_id] = r->mailbox;
2187                 }
2188         }
2189
2190         return 0;
2191 }
2192
2193 static void
2194 extern_func_build_free(struct rte_swx_pipeline *p)
2195 {
2196         uint32_t i;
2197
2198         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2199                 struct thread *t = &p->threads[i];
2200                 uint32_t j;
2201
2202                 if (!t->extern_funcs)
2203                         continue;
2204
2205                 for (j = 0; j < p->n_extern_funcs; j++) {
2206                         struct extern_func_runtime *r = &t->extern_funcs[j];
2207
2208                         free(r->mailbox);
2209                 }
2210
2211                 free(t->extern_funcs);
2212                 t->extern_funcs = NULL;
2213         }
2214 }
2215
2216 static void
2217 extern_func_free(struct rte_swx_pipeline *p)
2218 {
2219         extern_func_build_free(p);
2220
2221         for ( ; ; ) {
2222                 struct extern_func *elem;
2223
2224                 elem = TAILQ_FIRST(&p->extern_funcs);
2225                 if (!elem)
2226                         break;
2227
2228                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
2229                 free(elem);
2230         }
2231 }
2232
2233 /*
2234  * Header.
2235  */
2236 static struct header *
2237 header_find(struct rte_swx_pipeline *p, const char *name)
2238 {
2239         struct header *elem;
2240
2241         TAILQ_FOREACH(elem, &p->headers, node)
2242                 if (strcmp(elem->name, name) == 0)
2243                         return elem;
2244
2245         return NULL;
2246 }
2247
2248 static struct header *
2249 header_parse(struct rte_swx_pipeline *p,
2250              const char *name)
2251 {
2252         if (name[0] != 'h' || name[1] != '.')
2253                 return NULL;
2254
2255         return header_find(p, &name[2]);
2256 }
2257
2258 static struct field *
2259 header_field_parse(struct rte_swx_pipeline *p,
2260                    const char *name,
2261                    struct header **header)
2262 {
2263         struct header *h;
2264         struct field *f;
2265         char *header_name, *field_name;
2266
2267         if ((name[0] != 'h') || (name[1] != '.'))
2268                 return NULL;
2269
2270         header_name = strdup(&name[2]);
2271         if (!header_name)
2272                 return NULL;
2273
2274         field_name = strchr(header_name, '.');
2275         if (!field_name) {
2276                 free(header_name);
2277                 return NULL;
2278         }
2279
2280         *field_name = 0;
2281         field_name++;
2282
2283         h = header_find(p, header_name);
2284         if (!h) {
2285                 free(header_name);
2286                 return NULL;
2287         }
2288
2289         f = struct_type_field_find(h->st, field_name);
2290         if (!f) {
2291                 free(header_name);
2292                 return NULL;
2293         }
2294
2295         if (header)
2296                 *header = h;
2297
2298         free(header_name);
2299         return f;
2300 }
2301
2302 int
2303 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2304                                         const char *name,
2305                                         const char *struct_type_name)
2306 {
2307         struct struct_type *st;
2308         struct header *h;
2309         size_t n_headers_max;
2310
2311         CHECK(p, EINVAL);
2312         CHECK_NAME(name, EINVAL);
2313         CHECK_NAME(struct_type_name, EINVAL);
2314
2315         CHECK(!header_find(p, name), EEXIST);
2316
2317         st = struct_type_find(p, struct_type_name);
2318         CHECK(st, EINVAL);
2319
2320         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2321         CHECK(p->n_headers < n_headers_max, ENOSPC);
2322
2323         /* Node allocation. */
2324         h = calloc(1, sizeof(struct header));
2325         CHECK(h, ENOMEM);
2326
2327         /* Node initialization. */
2328         strcpy(h->name, name);
2329         h->st = st;
2330         h->struct_id = p->n_structs;
2331         h->id = p->n_headers;
2332
2333         /* Node add to tailq. */
2334         TAILQ_INSERT_TAIL(&p->headers, h, node);
2335         p->n_headers++;
2336         p->n_structs++;
2337
2338         return 0;
2339 }
2340
2341 static int
2342 header_build(struct rte_swx_pipeline *p)
2343 {
2344         struct header *h;
2345         uint32_t n_bytes = 0, i;
2346
2347         TAILQ_FOREACH(h, &p->headers, node) {
2348                 n_bytes += h->st->n_bits / 8;
2349         }
2350
2351         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2352                 struct thread *t = &p->threads[i];
2353                 uint32_t offset = 0;
2354
2355                 t->headers = calloc(p->n_headers,
2356                                     sizeof(struct header_runtime));
2357                 CHECK(t->headers, ENOMEM);
2358
2359                 t->headers_out = calloc(p->n_headers,
2360                                         sizeof(struct header_out_runtime));
2361                 CHECK(t->headers_out, ENOMEM);
2362
2363                 t->header_storage = calloc(1, n_bytes);
2364                 CHECK(t->header_storage, ENOMEM);
2365
2366                 t->header_out_storage = calloc(1, n_bytes);
2367                 CHECK(t->header_out_storage, ENOMEM);
2368
2369                 TAILQ_FOREACH(h, &p->headers, node) {
2370                         uint8_t *header_storage;
2371
2372                         header_storage = &t->header_storage[offset];
2373                         offset += h->st->n_bits / 8;
2374
2375                         t->headers[h->id].ptr0 = header_storage;
2376                         t->structs[h->struct_id] = header_storage;
2377                 }
2378         }
2379
2380         return 0;
2381 }
2382
2383 static void
2384 header_build_free(struct rte_swx_pipeline *p)
2385 {
2386         uint32_t i;
2387
2388         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2389                 struct thread *t = &p->threads[i];
2390
2391                 free(t->headers_out);
2392                 t->headers_out = NULL;
2393
2394                 free(t->headers);
2395                 t->headers = NULL;
2396
2397                 free(t->header_out_storage);
2398                 t->header_out_storage = NULL;
2399
2400                 free(t->header_storage);
2401                 t->header_storage = NULL;
2402         }
2403 }
2404
2405 static void
2406 header_free(struct rte_swx_pipeline *p)
2407 {
2408         header_build_free(p);
2409
2410         for ( ; ; ) {
2411                 struct header *elem;
2412
2413                 elem = TAILQ_FIRST(&p->headers);
2414                 if (!elem)
2415                         break;
2416
2417                 TAILQ_REMOVE(&p->headers, elem, node);
2418                 free(elem);
2419         }
2420 }
2421
2422 /*
2423  * Meta-data.
2424  */
2425 static struct field *
2426 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2427 {
2428         if (!p->metadata_st)
2429                 return NULL;
2430
2431         if (name[0] != 'm' || name[1] != '.')
2432                 return NULL;
2433
2434         return struct_type_field_find(p->metadata_st, &name[2]);
2435 }
2436
2437 int
2438 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2439                                           const char *struct_type_name)
2440 {
2441         struct struct_type *st = NULL;
2442
2443         CHECK(p, EINVAL);
2444
2445         CHECK_NAME(struct_type_name, EINVAL);
2446         st  = struct_type_find(p, struct_type_name);
2447         CHECK(st, EINVAL);
2448         CHECK(!p->metadata_st, EINVAL);
2449
2450         p->metadata_st = st;
2451         p->metadata_struct_id = p->n_structs;
2452
2453         p->n_structs++;
2454
2455         return 0;
2456 }
2457
2458 static int
2459 metadata_build(struct rte_swx_pipeline *p)
2460 {
2461         uint32_t n_bytes = p->metadata_st->n_bits / 8;
2462         uint32_t i;
2463
2464         /* Thread-level initialization. */
2465         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2466                 struct thread *t = &p->threads[i];
2467                 uint8_t *metadata;
2468
2469                 metadata = calloc(1, n_bytes);
2470                 CHECK(metadata, ENOMEM);
2471
2472                 t->metadata = metadata;
2473                 t->structs[p->metadata_struct_id] = metadata;
2474         }
2475
2476         return 0;
2477 }
2478
2479 static void
2480 metadata_build_free(struct rte_swx_pipeline *p)
2481 {
2482         uint32_t i;
2483
2484         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2485                 struct thread *t = &p->threads[i];
2486
2487                 free(t->metadata);
2488                 t->metadata = NULL;
2489         }
2490 }
2491
2492 static void
2493 metadata_free(struct rte_swx_pipeline *p)
2494 {
2495         metadata_build_free(p);
2496 }
2497
2498 /*
2499  * Instruction.
2500  */
2501 static int
2502 instruction_is_tx(enum instruction_type type)
2503 {
2504         switch (type) {
2505         case INSTR_TX:
2506         case INSTR_TX_I:
2507                 return 1;
2508
2509         default:
2510                 return 0;
2511         }
2512 }
2513
2514 static int
2515 instruction_is_jmp(struct instruction *instr)
2516 {
2517         switch (instr->type) {
2518         case INSTR_JMP:
2519         case INSTR_JMP_VALID:
2520         case INSTR_JMP_INVALID:
2521         case INSTR_JMP_HIT:
2522         case INSTR_JMP_MISS:
2523         case INSTR_JMP_ACTION_HIT:
2524         case INSTR_JMP_ACTION_MISS:
2525         case INSTR_JMP_EQ:
2526         case INSTR_JMP_EQ_S:
2527         case INSTR_JMP_EQ_I:
2528         case INSTR_JMP_NEQ:
2529         case INSTR_JMP_NEQ_S:
2530         case INSTR_JMP_NEQ_I:
2531         case INSTR_JMP_LT:
2532         case INSTR_JMP_LT_MH:
2533         case INSTR_JMP_LT_HM:
2534         case INSTR_JMP_LT_HH:
2535         case INSTR_JMP_LT_MI:
2536         case INSTR_JMP_LT_HI:
2537         case INSTR_JMP_GT:
2538         case INSTR_JMP_GT_MH:
2539         case INSTR_JMP_GT_HM:
2540         case INSTR_JMP_GT_HH:
2541         case INSTR_JMP_GT_MI:
2542         case INSTR_JMP_GT_HI:
2543                 return 1;
2544
2545         default:
2546                 return 0;
2547         }
2548 }
2549
2550 static struct field *
2551 action_field_parse(struct action *action, const char *name);
2552
2553 static struct field *
2554 struct_field_parse(struct rte_swx_pipeline *p,
2555                    struct action *action,
2556                    const char *name,
2557                    uint32_t *struct_id)
2558 {
2559         struct field *f;
2560
2561         switch (name[0]) {
2562         case 'h':
2563         {
2564                 struct header *header;
2565
2566                 f = header_field_parse(p, name, &header);
2567                 if (!f)
2568                         return NULL;
2569
2570                 *struct_id = header->struct_id;
2571                 return f;
2572         }
2573
2574         case 'm':
2575         {
2576                 f = metadata_field_parse(p, name);
2577                 if (!f)
2578                         return NULL;
2579
2580                 *struct_id = p->metadata_struct_id;
2581                 return f;
2582         }
2583
2584         case 't':
2585         {
2586                 if (!action)
2587                         return NULL;
2588
2589                 f = action_field_parse(action, name);
2590                 if (!f)
2591                         return NULL;
2592
2593                 *struct_id = 0;
2594                 return f;
2595         }
2596
2597         case 'e':
2598         {
2599                 struct extern_obj *obj;
2600
2601                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2602                 if (!f)
2603                         return NULL;
2604
2605                 *struct_id = obj->struct_id;
2606                 return f;
2607         }
2608
2609         case 'f':
2610         {
2611                 struct extern_func *func;
2612
2613                 f = extern_func_mailbox_field_parse(p, name, &func);
2614                 if (!f)
2615                         return NULL;
2616
2617                 *struct_id = func->struct_id;
2618                 return f;
2619         }
2620
2621         default:
2622                 return NULL;
2623         }
2624 }
2625
2626 static inline void
2627 pipeline_port_inc(struct rte_swx_pipeline *p)
2628 {
2629         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2630 }
2631
2632 static inline void
2633 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2634 {
2635         t->ip = p->instructions;
2636 }
2637
2638 static inline void
2639 thread_ip_set(struct thread *t, struct instruction *ip)
2640 {
2641         t->ip = ip;
2642 }
2643
2644 static inline void
2645 thread_ip_action_call(struct rte_swx_pipeline *p,
2646                       struct thread *t,
2647                       uint32_t action_id)
2648 {
2649         t->ret = t->ip + 1;
2650         t->ip = p->action_instructions[action_id];
2651 }
2652
2653 static inline void
2654 thread_ip_inc(struct rte_swx_pipeline *p);
2655
2656 static inline void
2657 thread_ip_inc(struct rte_swx_pipeline *p)
2658 {
2659         struct thread *t = &p->threads[p->thread_id];
2660
2661         t->ip++;
2662 }
2663
2664 static inline void
2665 thread_ip_inc_cond(struct thread *t, int cond)
2666 {
2667         t->ip += cond;
2668 }
2669
2670 static inline void
2671 thread_yield(struct rte_swx_pipeline *p)
2672 {
2673         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2674 }
2675
2676 static inline void
2677 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2678 {
2679         p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2680 }
2681
2682 /*
2683  * rx.
2684  */
2685 static int
2686 instr_rx_translate(struct rte_swx_pipeline *p,
2687                    struct action *action,
2688                    char **tokens,
2689                    int n_tokens,
2690                    struct instruction *instr,
2691                    struct instruction_data *data __rte_unused)
2692 {
2693         struct field *f;
2694
2695         CHECK(!action, EINVAL);
2696         CHECK(n_tokens == 2, EINVAL);
2697
2698         f = metadata_field_parse(p, tokens[1]);
2699         CHECK(f, EINVAL);
2700
2701         instr->type = INSTR_RX;
2702         instr->io.io.offset = f->offset / 8;
2703         instr->io.io.n_bits = f->n_bits;
2704         return 0;
2705 }
2706
2707 static inline void
2708 instr_rx_exec(struct rte_swx_pipeline *p);
2709
2710 static inline void
2711 instr_rx_exec(struct rte_swx_pipeline *p)
2712 {
2713         struct thread *t = &p->threads[p->thread_id];
2714         struct instruction *ip = t->ip;
2715         struct port_in_runtime *port = &p->in[p->port_id];
2716         struct rte_swx_pkt *pkt = &t->pkt;
2717         int pkt_received;
2718
2719         /* Packet. */
2720         pkt_received = port->pkt_rx(port->obj, pkt);
2721         t->ptr = &pkt->pkt[pkt->offset];
2722         rte_prefetch0(t->ptr);
2723
2724         TRACE("[Thread %2u] rx %s from port %u\n",
2725               p->thread_id,
2726               pkt_received ? "1 pkt" : "0 pkts",
2727               p->port_id);
2728
2729         /* Headers. */
2730         t->valid_headers = 0;
2731         t->n_headers_out = 0;
2732
2733         /* Meta-data. */
2734         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2735
2736         /* Tables. */
2737         t->table_state = p->table_state;
2738
2739         /* Thread. */
2740         pipeline_port_inc(p);
2741         thread_ip_inc_cond(t, pkt_received);
2742         thread_yield(p);
2743 }
2744
2745 /*
2746  * tx.
2747  */
2748 static int
2749 instr_tx_translate(struct rte_swx_pipeline *p,
2750                    struct action *action __rte_unused,
2751                    char **tokens,
2752                    int n_tokens,
2753                    struct instruction *instr,
2754                    struct instruction_data *data __rte_unused)
2755 {
2756         char *port = tokens[1];
2757         struct field *f;
2758         uint32_t port_val;
2759
2760         CHECK(n_tokens == 2, EINVAL);
2761
2762         f = metadata_field_parse(p, port);
2763         if (f) {
2764                 instr->type = INSTR_TX;
2765                 instr->io.io.offset = f->offset / 8;
2766                 instr->io.io.n_bits = f->n_bits;
2767                 return 0;
2768         }
2769
2770         /* TX_I. */
2771         port_val = strtoul(port, &port, 0);
2772         CHECK(!port[0], EINVAL);
2773
2774         instr->type = INSTR_TX_I;
2775         instr->io.io.val = port_val;
2776         return 0;
2777 }
2778
2779 static int
2780 instr_drop_translate(struct rte_swx_pipeline *p,
2781                      struct action *action __rte_unused,
2782                      char **tokens __rte_unused,
2783                      int n_tokens,
2784                      struct instruction *instr,
2785                      struct instruction_data *data __rte_unused)
2786 {
2787         CHECK(n_tokens == 1, EINVAL);
2788
2789         /* TX_I. */
2790         instr->type = INSTR_TX_I;
2791         instr->io.io.val = p->n_ports_out - 1;
2792         return 0;
2793 }
2794
2795 static inline void
2796 emit_handler(struct thread *t)
2797 {
2798         struct header_out_runtime *h0 = &t->headers_out[0];
2799         struct header_out_runtime *h1 = &t->headers_out[1];
2800         uint32_t offset = 0, i;
2801
2802         /* No header change or header decapsulation. */
2803         if ((t->n_headers_out == 1) &&
2804             (h0->ptr + h0->n_bytes == t->ptr)) {
2805                 TRACE("Emit handler: no header change or header decap.\n");
2806
2807                 t->pkt.offset -= h0->n_bytes;
2808                 t->pkt.length += h0->n_bytes;
2809
2810                 return;
2811         }
2812
2813         /* Header encapsulation (optionally, with prior header decasulation). */
2814         if ((t->n_headers_out == 2) &&
2815             (h1->ptr + h1->n_bytes == t->ptr) &&
2816             (h0->ptr == h0->ptr0)) {
2817                 uint32_t offset;
2818
2819                 TRACE("Emit handler: header encapsulation.\n");
2820
2821                 offset = h0->n_bytes + h1->n_bytes;
2822                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2823                 t->pkt.offset -= offset;
2824                 t->pkt.length += offset;
2825
2826                 return;
2827         }
2828
2829         /* Header insertion. */
2830         /* TBD */
2831
2832         /* Header extraction. */
2833         /* TBD */
2834
2835         /* For any other case. */
2836         TRACE("Emit handler: complex case.\n");
2837
2838         for (i = 0; i < t->n_headers_out; i++) {
2839                 struct header_out_runtime *h = &t->headers_out[i];
2840
2841                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2842                 offset += h->n_bytes;
2843         }
2844
2845         if (offset) {
2846                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2847                 t->pkt.offset -= offset;
2848                 t->pkt.length += offset;
2849         }
2850 }
2851
2852 static inline void
2853 instr_tx_exec(struct rte_swx_pipeline *p);
2854
2855 static inline void
2856 instr_tx_exec(struct rte_swx_pipeline *p)
2857 {
2858         struct thread *t = &p->threads[p->thread_id];
2859         struct instruction *ip = t->ip;
2860         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2861         struct port_out_runtime *port = &p->out[port_id];
2862         struct rte_swx_pkt *pkt = &t->pkt;
2863
2864         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2865               p->thread_id,
2866               (uint32_t)port_id);
2867
2868         /* Headers. */
2869         emit_handler(t);
2870
2871         /* Packet. */
2872         port->pkt_tx(port->obj, pkt);
2873
2874         /* Thread. */
2875         thread_ip_reset(p, t);
2876         instr_rx_exec(p);
2877 }
2878
2879 static inline void
2880 instr_tx_i_exec(struct rte_swx_pipeline *p)
2881 {
2882         struct thread *t = &p->threads[p->thread_id];
2883         struct instruction *ip = t->ip;
2884         uint64_t port_id = ip->io.io.val;
2885         struct port_out_runtime *port = &p->out[port_id];
2886         struct rte_swx_pkt *pkt = &t->pkt;
2887
2888         TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
2889               p->thread_id,
2890               (uint32_t)port_id);
2891
2892         /* Headers. */
2893         emit_handler(t);
2894
2895         /* Packet. */
2896         port->pkt_tx(port->obj, pkt);
2897
2898         /* Thread. */
2899         thread_ip_reset(p, t);
2900         instr_rx_exec(p);
2901 }
2902
2903 /*
2904  * extract.
2905  */
2906 static int
2907 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2908                             struct action *action,
2909                             char **tokens,
2910                             int n_tokens,
2911                             struct instruction *instr,
2912                             struct instruction_data *data __rte_unused)
2913 {
2914         struct header *h;
2915
2916         CHECK(!action, EINVAL);
2917         CHECK(n_tokens == 2, EINVAL);
2918
2919         h = header_parse(p, tokens[1]);
2920         CHECK(h, EINVAL);
2921
2922         instr->type = INSTR_HDR_EXTRACT;
2923         instr->io.hdr.header_id[0] = h->id;
2924         instr->io.hdr.struct_id[0] = h->struct_id;
2925         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2926         return 0;
2927 }
2928
2929 static inline void
2930 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2931
2932 static inline void
2933 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2934 {
2935         struct thread *t = &p->threads[p->thread_id];
2936         struct instruction *ip = t->ip;
2937         uint64_t valid_headers = t->valid_headers;
2938         uint8_t *ptr = t->ptr;
2939         uint32_t offset = t->pkt.offset;
2940         uint32_t length = t->pkt.length;
2941         uint32_t i;
2942
2943         for (i = 0; i < n_extract; i++) {
2944                 uint32_t header_id = ip->io.hdr.header_id[i];
2945                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2946                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2947
2948                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2949                       p->thread_id,
2950                       header_id,
2951                       n_bytes);
2952
2953                 /* Headers. */
2954                 t->structs[struct_id] = ptr;
2955                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2956
2957                 /* Packet. */
2958                 offset += n_bytes;
2959                 length -= n_bytes;
2960                 ptr += n_bytes;
2961         }
2962
2963         /* Headers. */
2964         t->valid_headers = valid_headers;
2965
2966         /* Packet. */
2967         t->pkt.offset = offset;
2968         t->pkt.length = length;
2969         t->ptr = ptr;
2970 }
2971
2972 static inline void
2973 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2974 {
2975         __instr_hdr_extract_exec(p, 1);
2976
2977         /* Thread. */
2978         thread_ip_inc(p);
2979 }
2980
2981 static inline void
2982 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2983 {
2984         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2985               p->thread_id);
2986
2987         __instr_hdr_extract_exec(p, 2);
2988
2989         /* Thread. */
2990         thread_ip_inc(p);
2991 }
2992
2993 static inline void
2994 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2995 {
2996         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2997               p->thread_id);
2998
2999         __instr_hdr_extract_exec(p, 3);
3000
3001         /* Thread. */
3002         thread_ip_inc(p);
3003 }
3004
3005 static inline void
3006 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
3007 {
3008         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3009               p->thread_id);
3010
3011         __instr_hdr_extract_exec(p, 4);
3012
3013         /* Thread. */
3014         thread_ip_inc(p);
3015 }
3016
3017 static inline void
3018 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
3019 {
3020         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3021               p->thread_id);
3022
3023         __instr_hdr_extract_exec(p, 5);
3024
3025         /* Thread. */
3026         thread_ip_inc(p);
3027 }
3028
3029 static inline void
3030 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
3031 {
3032         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3033               p->thread_id);
3034
3035         __instr_hdr_extract_exec(p, 6);
3036
3037         /* Thread. */
3038         thread_ip_inc(p);
3039 }
3040
3041 static inline void
3042 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
3043 {
3044         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3045               p->thread_id);
3046
3047         __instr_hdr_extract_exec(p, 7);
3048
3049         /* Thread. */
3050         thread_ip_inc(p);
3051 }
3052
3053 static inline void
3054 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
3055 {
3056         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3057               p->thread_id);
3058
3059         __instr_hdr_extract_exec(p, 8);
3060
3061         /* Thread. */
3062         thread_ip_inc(p);
3063 }
3064
3065 /*
3066  * emit.
3067  */
3068 static int
3069 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
3070                          struct action *action __rte_unused,
3071                          char **tokens,
3072                          int n_tokens,
3073                          struct instruction *instr,
3074                          struct instruction_data *data __rte_unused)
3075 {
3076         struct header *h;
3077
3078         CHECK(n_tokens == 2, EINVAL);
3079
3080         h = header_parse(p, tokens[1]);
3081         CHECK(h, EINVAL);
3082
3083         instr->type = INSTR_HDR_EMIT;
3084         instr->io.hdr.header_id[0] = h->id;
3085         instr->io.hdr.struct_id[0] = h->struct_id;
3086         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3087         return 0;
3088 }
3089
3090 static inline void
3091 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
3092
3093 static inline void
3094 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
3095 {
3096         struct thread *t = &p->threads[p->thread_id];
3097         struct instruction *ip = t->ip;
3098         uint32_t n_headers_out = t->n_headers_out;
3099         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
3100         uint8_t *ho_ptr = NULL;
3101         uint32_t ho_nbytes = 0, i;
3102
3103         for (i = 0; i < n_emit; i++) {
3104                 uint32_t header_id = ip->io.hdr.header_id[i];
3105                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3106                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3107
3108                 struct header_runtime *hi = &t->headers[header_id];
3109                 uint8_t *hi_ptr = t->structs[struct_id];
3110
3111                 TRACE("[Thread %2u]: emit header %u\n",
3112                       p->thread_id,
3113                       header_id);
3114
3115                 /* Headers. */
3116                 if (!i) {
3117                         if (!t->n_headers_out) {
3118                                 ho = &t->headers_out[0];
3119
3120                                 ho->ptr0 = hi->ptr0;
3121                                 ho->ptr = hi_ptr;
3122
3123                                 ho_ptr = hi_ptr;
3124                                 ho_nbytes = n_bytes;
3125
3126                                 n_headers_out = 1;
3127
3128                                 continue;
3129                         } else {
3130                                 ho_ptr = ho->ptr;
3131                                 ho_nbytes = ho->n_bytes;
3132                         }
3133                 }
3134
3135                 if (ho_ptr + ho_nbytes == hi_ptr) {
3136                         ho_nbytes += n_bytes;
3137                 } else {
3138                         ho->n_bytes = ho_nbytes;
3139
3140                         ho++;
3141                         ho->ptr0 = hi->ptr0;
3142                         ho->ptr = hi_ptr;
3143
3144                         ho_ptr = hi_ptr;
3145                         ho_nbytes = n_bytes;
3146
3147                         n_headers_out++;
3148                 }
3149         }
3150
3151         ho->n_bytes = ho_nbytes;
3152         t->n_headers_out = n_headers_out;
3153 }
3154
3155 static inline void
3156 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
3157 {
3158         __instr_hdr_emit_exec(p, 1);
3159
3160         /* Thread. */
3161         thread_ip_inc(p);
3162 }
3163
3164 static inline void
3165 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
3166 {
3167         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3168               p->thread_id);
3169
3170         __instr_hdr_emit_exec(p, 1);
3171         instr_tx_exec(p);
3172 }
3173
3174 static inline void
3175 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
3176 {
3177         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3178               p->thread_id);
3179
3180         __instr_hdr_emit_exec(p, 2);
3181         instr_tx_exec(p);
3182 }
3183
3184 static inline void
3185 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
3186 {
3187         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3188               p->thread_id);
3189
3190         __instr_hdr_emit_exec(p, 3);
3191         instr_tx_exec(p);
3192 }
3193
3194 static inline void
3195 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
3196 {
3197         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3198               p->thread_id);
3199
3200         __instr_hdr_emit_exec(p, 4);
3201         instr_tx_exec(p);
3202 }
3203
3204 static inline void
3205 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
3206 {
3207         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3208               p->thread_id);
3209
3210         __instr_hdr_emit_exec(p, 5);
3211         instr_tx_exec(p);
3212 }
3213
3214 static inline void
3215 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
3216 {
3217         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3218               p->thread_id);
3219
3220         __instr_hdr_emit_exec(p, 6);
3221         instr_tx_exec(p);
3222 }
3223
3224 static inline void
3225 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
3226 {
3227         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3228               p->thread_id);
3229
3230         __instr_hdr_emit_exec(p, 7);
3231         instr_tx_exec(p);
3232 }
3233
3234 static inline void
3235 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
3236 {
3237         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
3238               p->thread_id);
3239
3240         __instr_hdr_emit_exec(p, 8);
3241         instr_tx_exec(p);
3242 }
3243
3244 /*
3245  * validate.
3246  */
3247 static int
3248 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
3249                              struct action *action __rte_unused,
3250                              char **tokens,
3251                              int n_tokens,
3252                              struct instruction *instr,
3253                              struct instruction_data *data __rte_unused)
3254 {
3255         struct header *h;
3256
3257         CHECK(n_tokens == 2, EINVAL);
3258
3259         h = header_parse(p, tokens[1]);
3260         CHECK(h, EINVAL);
3261
3262         instr->type = INSTR_HDR_VALIDATE;
3263         instr->valid.header_id = h->id;
3264         return 0;
3265 }
3266
3267 static inline void
3268 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
3269 {
3270         struct thread *t = &p->threads[p->thread_id];
3271         struct instruction *ip = t->ip;
3272         uint32_t header_id = ip->valid.header_id;
3273
3274         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
3275
3276         /* Headers. */
3277         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
3278
3279         /* Thread. */
3280         thread_ip_inc(p);
3281 }
3282
3283 /*
3284  * invalidate.
3285  */
3286 static int
3287 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
3288                                struct action *action __rte_unused,
3289                                char **tokens,
3290                                int n_tokens,
3291                                struct instruction *instr,
3292                                struct instruction_data *data __rte_unused)
3293 {
3294         struct header *h;
3295
3296         CHECK(n_tokens == 2, EINVAL);
3297
3298         h = header_parse(p, tokens[1]);
3299         CHECK(h, EINVAL);
3300
3301         instr->type = INSTR_HDR_INVALIDATE;
3302         instr->valid.header_id = h->id;
3303         return 0;
3304 }
3305
3306 static inline void
3307 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3308 {
3309         struct thread *t = &p->threads[p->thread_id];
3310         struct instruction *ip = t->ip;
3311         uint32_t header_id = ip->valid.header_id;
3312
3313         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3314
3315         /* Headers. */
3316         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3317
3318         /* Thread. */
3319         thread_ip_inc(p);
3320 }
3321
3322 /*
3323  * table.
3324  */
3325 static struct table *
3326 table_find(struct rte_swx_pipeline *p, const char *name);
3327
3328 static int
3329 instr_table_translate(struct rte_swx_pipeline *p,
3330                       struct action *action,
3331                       char **tokens,
3332                       int n_tokens,
3333                       struct instruction *instr,
3334                       struct instruction_data *data __rte_unused)
3335 {
3336         struct table *t;
3337
3338         CHECK(!action, EINVAL);
3339         CHECK(n_tokens == 2, EINVAL);
3340
3341         t = table_find(p, tokens[1]);
3342         CHECK(t, EINVAL);
3343
3344         instr->type = INSTR_TABLE;
3345         instr->table.table_id = t->id;
3346         return 0;
3347 }
3348
3349 static inline void
3350 instr_table_exec(struct rte_swx_pipeline *p)
3351 {
3352         struct thread *t = &p->threads[p->thread_id];
3353         struct instruction *ip = t->ip;
3354         uint32_t table_id = ip->table.table_id;
3355         struct rte_swx_table_state *ts = &t->table_state[table_id];
3356         struct table_runtime *table = &t->tables[table_id];
3357         uint64_t action_id;
3358         uint8_t *action_data;
3359         int done, hit;
3360
3361         /* Table. */
3362         done = table->func(ts->obj,
3363                            table->mailbox,
3364                            table->key,
3365                            &action_id,
3366                            &action_data,
3367                            &hit);
3368         if (!done) {
3369                 /* Thread. */
3370                 TRACE("[Thread %2u] table %u (not finalized)\n",
3371                       p->thread_id,
3372                       table_id);
3373
3374                 thread_yield(p);
3375                 return;
3376         }
3377
3378         action_id = hit ? action_id : ts->default_action_id;
3379         action_data = hit ? action_data : ts->default_action_data;
3380
3381         TRACE("[Thread %2u] table %u (%s, action %u)\n",
3382               p->thread_id,
3383               table_id,
3384               hit ? "hit" : "miss",
3385               (uint32_t)action_id);
3386
3387         t->action_id = action_id;
3388         t->structs[0] = action_data;
3389         t->hit = hit;
3390
3391         /* Thread. */
3392         thread_ip_action_call(p, t, action_id);
3393 }
3394
3395 /*
3396  * extern.
3397  */
3398 static int
3399 instr_extern_translate(struct rte_swx_pipeline *p,
3400                        struct action *action __rte_unused,
3401                        char **tokens,
3402                        int n_tokens,
3403                        struct instruction *instr,
3404                        struct instruction_data *data __rte_unused)
3405 {
3406         char *token = tokens[1];
3407
3408         CHECK(n_tokens == 2, EINVAL);
3409
3410         if (token[0] == 'e') {
3411                 struct extern_obj *obj;
3412                 struct extern_type_member_func *func;
3413
3414                 func = extern_obj_member_func_parse(p, token, &obj);
3415                 CHECK(func, EINVAL);
3416
3417                 instr->type = INSTR_EXTERN_OBJ;
3418                 instr->ext_obj.ext_obj_id = obj->id;
3419                 instr->ext_obj.func_id = func->id;
3420
3421                 return 0;
3422         }
3423
3424         if (token[0] == 'f') {
3425                 struct extern_func *func;
3426
3427                 func = extern_func_parse(p, token);
3428                 CHECK(func, EINVAL);
3429
3430                 instr->type = INSTR_EXTERN_FUNC;
3431                 instr->ext_func.ext_func_id = func->id;
3432
3433                 return 0;
3434         }
3435
3436         CHECK(0, EINVAL);
3437 }
3438
3439 static inline void
3440 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3441 {
3442         struct thread *t = &p->threads[p->thread_id];
3443         struct instruction *ip = t->ip;
3444         uint32_t obj_id = ip->ext_obj.ext_obj_id;
3445         uint32_t func_id = ip->ext_obj.func_id;
3446         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3447         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3448
3449         TRACE("[Thread %2u] extern obj %u member func %u\n",
3450               p->thread_id,
3451               obj_id,
3452               func_id);
3453
3454         /* Extern object member function execute. */
3455         uint32_t done = func(obj->obj, obj->mailbox);
3456
3457         /* Thread. */
3458         thread_ip_inc_cond(t, done);
3459         thread_yield_cond(p, done ^ 1);
3460 }
3461
3462 static inline void
3463 instr_extern_func_exec(struct rte_swx_pipeline *p)
3464 {
3465         struct thread *t = &p->threads[p->thread_id];
3466         struct instruction *ip = t->ip;
3467         uint32_t ext_func_id = ip->ext_func.ext_func_id;
3468         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3469         rte_swx_extern_func_t func = ext_func->func;
3470
3471         TRACE("[Thread %2u] extern func %u\n",
3472               p->thread_id,
3473               ext_func_id);
3474
3475         /* Extern function execute. */
3476         uint32_t done = func(ext_func->mailbox);
3477
3478         /* Thread. */
3479         thread_ip_inc_cond(t, done);
3480         thread_yield_cond(p, done ^ 1);
3481 }
3482
3483 /*
3484  * mov.
3485  */
3486 static int
3487 instr_mov_translate(struct rte_swx_pipeline *p,
3488                     struct action *action,
3489                     char **tokens,
3490                     int n_tokens,
3491                     struct instruction *instr,
3492                     struct instruction_data *data __rte_unused)
3493 {
3494         char *dst = tokens[1], *src = tokens[2];
3495         struct field *fdst, *fsrc;
3496         uint64_t src_val;
3497         uint32_t dst_struct_id, src_struct_id;
3498
3499         CHECK(n_tokens == 3, EINVAL);
3500
3501         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3502         CHECK(fdst, EINVAL);
3503
3504         /* MOV or MOV_S. */
3505         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3506         if (fsrc) {
3507                 instr->type = INSTR_MOV;
3508                 if ((dst[0] == 'h' && src[0] != 'h') ||
3509                     (dst[0] != 'h' && src[0] == 'h'))
3510                         instr->type = INSTR_MOV_S;
3511
3512                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3513                 instr->mov.dst.n_bits = fdst->n_bits;
3514                 instr->mov.dst.offset = fdst->offset / 8;
3515                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3516                 instr->mov.src.n_bits = fsrc->n_bits;
3517                 instr->mov.src.offset = fsrc->offset / 8;
3518                 return 0;
3519         }
3520
3521         /* MOV_I. */
3522         src_val = strtoull(src, &src, 0);
3523         CHECK(!src[0], EINVAL);
3524
3525         if (dst[0] == 'h')
3526                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3527
3528         instr->type = INSTR_MOV_I;
3529         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3530         instr->mov.dst.n_bits = fdst->n_bits;
3531         instr->mov.dst.offset = fdst->offset / 8;
3532         instr->mov.src_val = src_val;
3533         return 0;
3534 }
3535
3536 static inline void
3537 instr_mov_exec(struct rte_swx_pipeline *p)
3538 {
3539         struct thread *t = &p->threads[p->thread_id];
3540         struct instruction *ip = t->ip;
3541
3542         TRACE("[Thread %2u] mov\n",
3543               p->thread_id);
3544
3545         MOV(t, ip);
3546
3547         /* Thread. */
3548         thread_ip_inc(p);
3549 }
3550
3551 static inline void
3552 instr_mov_s_exec(struct rte_swx_pipeline *p)
3553 {
3554         struct thread *t = &p->threads[p->thread_id];
3555         struct instruction *ip = t->ip;
3556
3557         TRACE("[Thread %2u] mov (s)\n",
3558               p->thread_id);
3559
3560         MOV_S(t, ip);
3561
3562         /* Thread. */
3563         thread_ip_inc(p);
3564 }
3565
3566 static inline void
3567 instr_mov_i_exec(struct rte_swx_pipeline *p)
3568 {
3569         struct thread *t = &p->threads[p->thread_id];
3570         struct instruction *ip = t->ip;
3571
3572         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
3573               p->thread_id,
3574               ip->mov.src_val);
3575
3576         MOV_I(t, ip);
3577
3578         /* Thread. */
3579         thread_ip_inc(p);
3580 }
3581
3582 /*
3583  * dma.
3584  */
3585 static int
3586 instr_dma_translate(struct rte_swx_pipeline *p,
3587                     struct action *action,
3588                     char **tokens,
3589                     int n_tokens,
3590                     struct instruction *instr,
3591                     struct instruction_data *data __rte_unused)
3592 {
3593         char *dst = tokens[1];
3594         char *src = tokens[2];
3595         struct header *h;
3596         struct field *tf;
3597
3598         CHECK(action, EINVAL);
3599         CHECK(n_tokens == 3, EINVAL);
3600
3601         h = header_parse(p, dst);
3602         CHECK(h, EINVAL);
3603
3604         tf = action_field_parse(action, src);
3605         CHECK(tf, EINVAL);
3606
3607         instr->type = INSTR_DMA_HT;
3608         instr->dma.dst.header_id[0] = h->id;
3609         instr->dma.dst.struct_id[0] = h->struct_id;
3610         instr->dma.n_bytes[0] = h->st->n_bits / 8;
3611         instr->dma.src.offset[0] = tf->offset / 8;
3612
3613         return 0;
3614 }
3615
3616 static inline void
3617 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3618
3619 static inline void
3620 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3621 {
3622         struct thread *t = &p->threads[p->thread_id];
3623         struct instruction *ip = t->ip;
3624         uint8_t *action_data = t->structs[0];
3625         uint64_t valid_headers = t->valid_headers;
3626         uint32_t i;
3627
3628         for (i = 0; i < n_dma; i++) {
3629                 uint32_t header_id = ip->dma.dst.header_id[i];
3630                 uint32_t struct_id = ip->dma.dst.struct_id[i];
3631                 uint32_t offset = ip->dma.src.offset[i];
3632                 uint32_t n_bytes = ip->dma.n_bytes[i];
3633
3634                 struct header_runtime *h = &t->headers[header_id];
3635                 uint8_t *h_ptr0 = h->ptr0;
3636                 uint8_t *h_ptr = t->structs[struct_id];
3637
3638                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3639                         h_ptr : h_ptr0;
3640                 void *src = &action_data[offset];
3641
3642                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3643
3644                 /* Headers. */
3645                 memcpy(dst, src, n_bytes);
3646                 t->structs[struct_id] = dst;
3647                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3648         }
3649
3650         t->valid_headers = valid_headers;
3651 }
3652
3653 static inline void
3654 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3655 {
3656         __instr_dma_ht_exec(p, 1);
3657
3658         /* Thread. */
3659         thread_ip_inc(p);
3660 }
3661
3662 static inline void
3663 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3664 {
3665         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3666               p->thread_id);
3667
3668         __instr_dma_ht_exec(p, 2);
3669
3670         /* Thread. */
3671         thread_ip_inc(p);
3672 }
3673
3674 static inline void
3675 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3676 {
3677         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3678               p->thread_id);
3679
3680         __instr_dma_ht_exec(p, 3);
3681
3682         /* Thread. */
3683         thread_ip_inc(p);
3684 }
3685
3686 static inline void
3687 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3688 {
3689         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3690               p->thread_id);
3691
3692         __instr_dma_ht_exec(p, 4);
3693
3694         /* Thread. */
3695         thread_ip_inc(p);
3696 }
3697
3698 static inline void
3699 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3700 {
3701         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3702               p->thread_id);
3703
3704         __instr_dma_ht_exec(p, 5);
3705
3706         /* Thread. */
3707         thread_ip_inc(p);
3708 }
3709
3710 static inline void
3711 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3712 {
3713         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3714               p->thread_id);
3715
3716         __instr_dma_ht_exec(p, 6);
3717
3718         /* Thread. */
3719         thread_ip_inc(p);
3720 }
3721
3722 static inline void
3723 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3724 {
3725         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3726               p->thread_id);
3727
3728         __instr_dma_ht_exec(p, 7);
3729
3730         /* Thread. */
3731         thread_ip_inc(p);
3732 }
3733
3734 static inline void
3735 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3736 {
3737         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3738               p->thread_id);
3739
3740         __instr_dma_ht_exec(p, 8);
3741
3742         /* Thread. */
3743         thread_ip_inc(p);
3744 }
3745
3746 /*
3747  * alu.
3748  */
3749 static int
3750 instr_alu_add_translate(struct rte_swx_pipeline *p,
3751                         struct action *action,
3752                         char **tokens,
3753                         int n_tokens,
3754                         struct instruction *instr,
3755                         struct instruction_data *data __rte_unused)
3756 {
3757         char *dst = tokens[1], *src = tokens[2];
3758         struct field *fdst, *fsrc;
3759         uint64_t src_val;
3760         uint32_t dst_struct_id, src_struct_id;
3761
3762         CHECK(n_tokens == 3, EINVAL);
3763
3764         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3765         CHECK(fdst, EINVAL);
3766
3767         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3768         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3769         if (fsrc) {
3770                 instr->type = INSTR_ALU_ADD;
3771                 if (dst[0] == 'h' && src[0] != 'h')
3772                         instr->type = INSTR_ALU_ADD_HM;
3773                 if (dst[0] != 'h' && src[0] == 'h')
3774                         instr->type = INSTR_ALU_ADD_MH;
3775                 if (dst[0] == 'h' && src[0] == 'h')
3776                         instr->type = INSTR_ALU_ADD_HH;
3777
3778                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3779                 instr->alu.dst.n_bits = fdst->n_bits;
3780                 instr->alu.dst.offset = fdst->offset / 8;
3781                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3782                 instr->alu.src.n_bits = fsrc->n_bits;
3783                 instr->alu.src.offset = fsrc->offset / 8;
3784                 return 0;
3785         }
3786
3787         /* ADD_MI, ADD_HI. */
3788         src_val = strtoull(src, &src, 0);
3789         CHECK(!src[0], EINVAL);
3790
3791         instr->type = INSTR_ALU_ADD_MI;
3792         if (dst[0] == 'h')
3793                 instr->type = INSTR_ALU_ADD_HI;
3794
3795         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3796         instr->alu.dst.n_bits = fdst->n_bits;
3797         instr->alu.dst.offset = fdst->offset / 8;
3798         instr->alu.src_val = src_val;
3799         return 0;
3800 }
3801
3802 static int
3803 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3804                         struct action *action,
3805                         char **tokens,
3806                         int n_tokens,
3807                         struct instruction *instr,
3808                         struct instruction_data *data __rte_unused)
3809 {
3810         char *dst = tokens[1], *src = tokens[2];
3811         struct field *fdst, *fsrc;
3812         uint64_t src_val;
3813         uint32_t dst_struct_id, src_struct_id;
3814
3815         CHECK(n_tokens == 3, EINVAL);
3816
3817         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3818         CHECK(fdst, EINVAL);
3819
3820         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3821         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3822         if (fsrc) {
3823                 instr->type = INSTR_ALU_SUB;
3824                 if (dst[0] == 'h' && src[0] != 'h')
3825                         instr->type = INSTR_ALU_SUB_HM;
3826                 if (dst[0] != 'h' && src[0] == 'h')
3827                         instr->type = INSTR_ALU_SUB_MH;
3828                 if (dst[0] == 'h' && src[0] == 'h')
3829                         instr->type = INSTR_ALU_SUB_HH;
3830
3831                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3832                 instr->alu.dst.n_bits = fdst->n_bits;
3833                 instr->alu.dst.offset = fdst->offset / 8;
3834                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3835                 instr->alu.src.n_bits = fsrc->n_bits;
3836                 instr->alu.src.offset = fsrc->offset / 8;
3837                 return 0;
3838         }
3839
3840         /* SUB_MI, SUB_HI. */
3841         src_val = strtoull(src, &src, 0);
3842         CHECK(!src[0], EINVAL);
3843
3844         instr->type = INSTR_ALU_SUB_MI;
3845         if (dst[0] == 'h')
3846                 instr->type = INSTR_ALU_SUB_HI;
3847
3848         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3849         instr->alu.dst.n_bits = fdst->n_bits;
3850         instr->alu.dst.offset = fdst->offset / 8;
3851         instr->alu.src_val = src_val;
3852         return 0;
3853 }
3854
3855 static int
3856 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3857                           struct action *action __rte_unused,
3858                           char **tokens,
3859                           int n_tokens,
3860                           struct instruction *instr,
3861                           struct instruction_data *data __rte_unused)
3862 {
3863         char *dst = tokens[1], *src = tokens[2];
3864         struct header *hdst, *hsrc;
3865         struct field *fdst, *fsrc;
3866
3867         CHECK(n_tokens == 3, EINVAL);
3868
3869         fdst = header_field_parse(p, dst, &hdst);
3870         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3871
3872         /* CKADD_FIELD. */
3873         fsrc = header_field_parse(p, src, &hsrc);
3874         if (fsrc) {
3875                 instr->type = INSTR_ALU_CKADD_FIELD;
3876                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3877                 instr->alu.dst.n_bits = fdst->n_bits;
3878                 instr->alu.dst.offset = fdst->offset / 8;
3879                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3880                 instr->alu.src.n_bits = fsrc->n_bits;
3881                 instr->alu.src.offset = fsrc->offset / 8;
3882                 return 0;
3883         }
3884
3885         /* CKADD_STRUCT, CKADD_STRUCT20. */
3886         hsrc = header_parse(p, src);
3887         CHECK(hsrc, EINVAL);
3888
3889         instr->type = INSTR_ALU_CKADD_STRUCT;
3890         if ((hsrc->st->n_bits / 8) == 20)
3891                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3892
3893         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3894         instr->alu.dst.n_bits = fdst->n_bits;
3895         instr->alu.dst.offset = fdst->offset / 8;
3896         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3897         instr->alu.src.n_bits = hsrc->st->n_bits;
3898         instr->alu.src.offset = 0; /* Unused. */
3899         return 0;
3900 }
3901
3902 static int
3903 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3904                           struct action *action __rte_unused,
3905                           char **tokens,
3906                           int n_tokens,
3907                           struct instruction *instr,
3908                           struct instruction_data *data __rte_unused)
3909 {
3910         char *dst = tokens[1], *src = tokens[2];
3911         struct header *hdst, *hsrc;
3912         struct field *fdst, *fsrc;
3913
3914         CHECK(n_tokens == 3, EINVAL);
3915
3916         fdst = header_field_parse(p, dst, &hdst);
3917         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3918
3919         fsrc = header_field_parse(p, src, &hsrc);
3920         CHECK(fsrc, EINVAL);
3921
3922         instr->type = INSTR_ALU_CKSUB_FIELD;
3923         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3924         instr->alu.dst.n_bits = fdst->n_bits;
3925         instr->alu.dst.offset = fdst->offset / 8;
3926         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3927         instr->alu.src.n_bits = fsrc->n_bits;
3928         instr->alu.src.offset = fsrc->offset / 8;
3929         return 0;
3930 }
3931
3932 static int
3933 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3934                         struct action *action,
3935                         char **tokens,
3936                         int n_tokens,
3937                         struct instruction *instr,
3938                         struct instruction_data *data __rte_unused)
3939 {
3940         char *dst = tokens[1], *src = tokens[2];
3941         struct field *fdst, *fsrc;
3942         uint64_t src_val;
3943         uint32_t dst_struct_id, src_struct_id;
3944
3945         CHECK(n_tokens == 3, EINVAL);
3946
3947         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3948         CHECK(fdst, EINVAL);
3949
3950         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3951         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3952         if (fsrc) {
3953                 instr->type = INSTR_ALU_SHL;
3954                 if (dst[0] == 'h' && src[0] != 'h')
3955                         instr->type = INSTR_ALU_SHL_HM;
3956                 if (dst[0] != 'h' && src[0] == 'h')
3957                         instr->type = INSTR_ALU_SHL_MH;
3958                 if (dst[0] == 'h' && src[0] == 'h')
3959                         instr->type = INSTR_ALU_SHL_HH;
3960
3961                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3962                 instr->alu.dst.n_bits = fdst->n_bits;
3963                 instr->alu.dst.offset = fdst->offset / 8;
3964                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3965                 instr->alu.src.n_bits = fsrc->n_bits;
3966                 instr->alu.src.offset = fsrc->offset / 8;
3967                 return 0;
3968         }
3969
3970         /* SHL_MI, SHL_HI. */
3971         src_val = strtoull(src, &src, 0);
3972         CHECK(!src[0], EINVAL);
3973
3974         instr->type = INSTR_ALU_SHL_MI;
3975         if (dst[0] == 'h')
3976                 instr->type = INSTR_ALU_SHL_HI;
3977
3978         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3979         instr->alu.dst.n_bits = fdst->n_bits;
3980         instr->alu.dst.offset = fdst->offset / 8;
3981         instr->alu.src_val = src_val;
3982         return 0;
3983 }
3984
3985 static int
3986 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3987                         struct action *action,
3988                         char **tokens,
3989                         int n_tokens,
3990                         struct instruction *instr,
3991                         struct instruction_data *data __rte_unused)
3992 {
3993         char *dst = tokens[1], *src = tokens[2];
3994         struct field *fdst, *fsrc;
3995         uint64_t src_val;
3996         uint32_t dst_struct_id, src_struct_id;
3997
3998         CHECK(n_tokens == 3, EINVAL);
3999
4000         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4001         CHECK(fdst, EINVAL);
4002
4003         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
4004         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4005         if (fsrc) {
4006                 instr->type = INSTR_ALU_SHR;
4007                 if (dst[0] == 'h' && src[0] != 'h')
4008                         instr->type = INSTR_ALU_SHR_HM;
4009                 if (dst[0] != 'h' && src[0] == 'h')
4010                         instr->type = INSTR_ALU_SHR_MH;
4011                 if (dst[0] == 'h' && src[0] == 'h')
4012                         instr->type = INSTR_ALU_SHR_HH;
4013
4014                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4015                 instr->alu.dst.n_bits = fdst->n_bits;
4016                 instr->alu.dst.offset = fdst->offset / 8;
4017                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4018                 instr->alu.src.n_bits = fsrc->n_bits;
4019                 instr->alu.src.offset = fsrc->offset / 8;
4020                 return 0;
4021         }
4022
4023         /* SHR_MI, SHR_HI. */
4024         src_val = strtoull(src, &src, 0);
4025         CHECK(!src[0], EINVAL);
4026
4027         instr->type = INSTR_ALU_SHR_MI;
4028         if (dst[0] == 'h')
4029                 instr->type = INSTR_ALU_SHR_HI;
4030
4031         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4032         instr->alu.dst.n_bits = fdst->n_bits;
4033         instr->alu.dst.offset = fdst->offset / 8;
4034         instr->alu.src_val = src_val;
4035         return 0;
4036 }
4037
4038 static int
4039 instr_alu_and_translate(struct rte_swx_pipeline *p,
4040                         struct action *action,
4041                         char **tokens,
4042                         int n_tokens,
4043                         struct instruction *instr,
4044                         struct instruction_data *data __rte_unused)
4045 {
4046         char *dst = tokens[1], *src = tokens[2];
4047         struct field *fdst, *fsrc;
4048         uint64_t src_val;
4049         uint32_t dst_struct_id, src_struct_id;
4050
4051         CHECK(n_tokens == 3, EINVAL);
4052
4053         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4054         CHECK(fdst, EINVAL);
4055
4056         /* AND or AND_S. */
4057         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4058         if (fsrc) {
4059                 instr->type = INSTR_ALU_AND;
4060                 if ((dst[0] == 'h' && src[0] != 'h') ||
4061                     (dst[0] != 'h' && src[0] == 'h'))
4062                         instr->type = INSTR_ALU_AND_S;
4063
4064                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4065                 instr->alu.dst.n_bits = fdst->n_bits;
4066                 instr->alu.dst.offset = fdst->offset / 8;
4067                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4068                 instr->alu.src.n_bits = fsrc->n_bits;
4069                 instr->alu.src.offset = fsrc->offset / 8;
4070                 return 0;
4071         }
4072
4073         /* AND_I. */
4074         src_val = strtoull(src, &src, 0);
4075         CHECK(!src[0], EINVAL);
4076
4077         if (dst[0] == 'h')
4078                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4079
4080         instr->type = INSTR_ALU_AND_I;
4081         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4082         instr->alu.dst.n_bits = fdst->n_bits;
4083         instr->alu.dst.offset = fdst->offset / 8;
4084         instr->alu.src_val = src_val;
4085         return 0;
4086 }
4087
4088 static int
4089 instr_alu_or_translate(struct rte_swx_pipeline *p,
4090                        struct action *action,
4091                        char **tokens,
4092                        int n_tokens,
4093                        struct instruction *instr,
4094                        struct instruction_data *data __rte_unused)
4095 {
4096         char *dst = tokens[1], *src = tokens[2];
4097         struct field *fdst, *fsrc;
4098         uint64_t src_val;
4099         uint32_t dst_struct_id, src_struct_id;
4100
4101         CHECK(n_tokens == 3, EINVAL);
4102
4103         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4104         CHECK(fdst, EINVAL);
4105
4106         /* OR or OR_S. */
4107         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4108         if (fsrc) {
4109                 instr->type = INSTR_ALU_OR;
4110                 if ((dst[0] == 'h' && src[0] != 'h') ||
4111                     (dst[0] != 'h' && src[0] == 'h'))
4112                         instr->type = INSTR_ALU_OR_S;
4113
4114                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4115                 instr->alu.dst.n_bits = fdst->n_bits;
4116                 instr->alu.dst.offset = fdst->offset / 8;
4117                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4118                 instr->alu.src.n_bits = fsrc->n_bits;
4119                 instr->alu.src.offset = fsrc->offset / 8;
4120                 return 0;
4121         }
4122
4123         /* OR_I. */
4124         src_val = strtoull(src, &src, 0);
4125         CHECK(!src[0], EINVAL);
4126
4127         if (dst[0] == 'h')
4128                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4129
4130         instr->type = INSTR_ALU_OR_I;
4131         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4132         instr->alu.dst.n_bits = fdst->n_bits;
4133         instr->alu.dst.offset = fdst->offset / 8;
4134         instr->alu.src_val = src_val;
4135         return 0;
4136 }
4137
4138 static int
4139 instr_alu_xor_translate(struct rte_swx_pipeline *p,
4140                         struct action *action,
4141                         char **tokens,
4142                         int n_tokens,
4143                         struct instruction *instr,
4144                         struct instruction_data *data __rte_unused)
4145 {
4146         char *dst = tokens[1], *src = tokens[2];
4147         struct field *fdst, *fsrc;
4148         uint64_t src_val;
4149         uint32_t dst_struct_id, src_struct_id;
4150
4151         CHECK(n_tokens == 3, EINVAL);
4152
4153         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4154         CHECK(fdst, EINVAL);
4155
4156         /* XOR or XOR_S. */
4157         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4158         if (fsrc) {
4159                 instr->type = INSTR_ALU_XOR;
4160                 if ((dst[0] == 'h' && src[0] != 'h') ||
4161                     (dst[0] != 'h' && src[0] == 'h'))
4162                         instr->type = INSTR_ALU_XOR_S;
4163
4164                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4165                 instr->alu.dst.n_bits = fdst->n_bits;
4166                 instr->alu.dst.offset = fdst->offset / 8;
4167                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4168                 instr->alu.src.n_bits = fsrc->n_bits;
4169                 instr->alu.src.offset = fsrc->offset / 8;
4170                 return 0;
4171         }
4172
4173         /* XOR_I. */
4174         src_val = strtoull(src, &src, 0);
4175         CHECK(!src[0], EINVAL);
4176
4177         if (dst[0] == 'h')
4178                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4179
4180         instr->type = INSTR_ALU_XOR_I;
4181         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4182         instr->alu.dst.n_bits = fdst->n_bits;
4183         instr->alu.dst.offset = fdst->offset / 8;
4184         instr->alu.src_val = src_val;
4185         return 0;
4186 }
4187
4188 static inline void
4189 instr_alu_add_exec(struct rte_swx_pipeline *p)
4190 {
4191         struct thread *t = &p->threads[p->thread_id];
4192         struct instruction *ip = t->ip;
4193
4194         TRACE("[Thread %2u] add\n", p->thread_id);
4195
4196         /* Structs. */
4197         ALU(t, ip, +);
4198
4199         /* Thread. */
4200         thread_ip_inc(p);
4201 }
4202
4203 static inline void
4204 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
4205 {
4206         struct thread *t = &p->threads[p->thread_id];
4207         struct instruction *ip = t->ip;
4208
4209         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
4210
4211         /* Structs. */
4212         ALU_MH(t, ip, +);
4213
4214         /* Thread. */
4215         thread_ip_inc(p);
4216 }
4217
4218 static inline void
4219 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
4220 {
4221         struct thread *t = &p->threads[p->thread_id];
4222         struct instruction *ip = t->ip;
4223
4224         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
4225
4226         /* Structs. */
4227         ALU_HM(t, ip, +);
4228
4229         /* Thread. */
4230         thread_ip_inc(p);
4231 }
4232
4233 static inline void
4234 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
4235 {
4236         struct thread *t = &p->threads[p->thread_id];
4237         struct instruction *ip = t->ip;
4238
4239         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
4240
4241         /* Structs. */
4242         ALU_HH(t, ip, +);
4243
4244         /* Thread. */
4245         thread_ip_inc(p);
4246 }
4247
4248 static inline void
4249 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
4250 {
4251         struct thread *t = &p->threads[p->thread_id];
4252         struct instruction *ip = t->ip;
4253
4254         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
4255
4256         /* Structs. */
4257         ALU_MI(t, ip, +);
4258
4259         /* Thread. */
4260         thread_ip_inc(p);
4261 }
4262
4263 static inline void
4264 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
4265 {
4266         struct thread *t = &p->threads[p->thread_id];
4267         struct instruction *ip = t->ip;
4268
4269         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
4270
4271         /* Structs. */
4272         ALU_HI(t, ip, +);
4273
4274         /* Thread. */
4275         thread_ip_inc(p);
4276 }
4277
4278 static inline void
4279 instr_alu_sub_exec(struct rte_swx_pipeline *p)
4280 {
4281         struct thread *t = &p->threads[p->thread_id];
4282         struct instruction *ip = t->ip;
4283
4284         TRACE("[Thread %2u] sub\n", p->thread_id);
4285
4286         /* Structs. */
4287         ALU(t, ip, -);
4288
4289         /* Thread. */
4290         thread_ip_inc(p);
4291 }
4292
4293 static inline void
4294 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
4295 {
4296         struct thread *t = &p->threads[p->thread_id];
4297         struct instruction *ip = t->ip;
4298
4299         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
4300
4301         /* Structs. */
4302         ALU_MH(t, ip, -);
4303
4304         /* Thread. */
4305         thread_ip_inc(p);
4306 }
4307
4308 static inline void
4309 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4310 {
4311         struct thread *t = &p->threads[p->thread_id];
4312         struct instruction *ip = t->ip;
4313
4314         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4315
4316         /* Structs. */
4317         ALU_HM(t, ip, -);
4318
4319         /* Thread. */
4320         thread_ip_inc(p);
4321 }
4322
4323 static inline void
4324 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4325 {
4326         struct thread *t = &p->threads[p->thread_id];
4327         struct instruction *ip = t->ip;
4328
4329         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4330
4331         /* Structs. */
4332         ALU_HH(t, ip, -);
4333
4334         /* Thread. */
4335         thread_ip_inc(p);
4336 }
4337
4338 static inline void
4339 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4340 {
4341         struct thread *t = &p->threads[p->thread_id];
4342         struct instruction *ip = t->ip;
4343
4344         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4345
4346         /* Structs. */
4347         ALU_MI(t, ip, -);
4348
4349         /* Thread. */
4350         thread_ip_inc(p);
4351 }
4352
4353 static inline void
4354 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4355 {
4356         struct thread *t = &p->threads[p->thread_id];
4357         struct instruction *ip = t->ip;
4358
4359         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4360
4361         /* Structs. */
4362         ALU_HI(t, ip, -);
4363
4364         /* Thread. */
4365         thread_ip_inc(p);
4366 }
4367
4368 static inline void
4369 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4370 {
4371         struct thread *t = &p->threads[p->thread_id];
4372         struct instruction *ip = t->ip;
4373
4374         TRACE("[Thread %2u] shl\n", p->thread_id);
4375
4376         /* Structs. */
4377         ALU(t, ip, <<);
4378
4379         /* Thread. */
4380         thread_ip_inc(p);
4381 }
4382
4383 static inline void
4384 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4385 {
4386         struct thread *t = &p->threads[p->thread_id];
4387         struct instruction *ip = t->ip;
4388
4389         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4390
4391         /* Structs. */
4392         ALU_MH(t, ip, <<);
4393
4394         /* Thread. */
4395         thread_ip_inc(p);
4396 }
4397
4398 static inline void
4399 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4400 {
4401         struct thread *t = &p->threads[p->thread_id];
4402         struct instruction *ip = t->ip;
4403
4404         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4405
4406         /* Structs. */
4407         ALU_HM(t, ip, <<);
4408
4409         /* Thread. */
4410         thread_ip_inc(p);
4411 }
4412
4413 static inline void
4414 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4415 {
4416         struct thread *t = &p->threads[p->thread_id];
4417         struct instruction *ip = t->ip;
4418
4419         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4420
4421         /* Structs. */
4422         ALU_HH(t, ip, <<);
4423
4424         /* Thread. */
4425         thread_ip_inc(p);
4426 }
4427
4428 static inline void
4429 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4430 {
4431         struct thread *t = &p->threads[p->thread_id];
4432         struct instruction *ip = t->ip;
4433
4434         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4435
4436         /* Structs. */
4437         ALU_MI(t, ip, <<);
4438
4439         /* Thread. */
4440         thread_ip_inc(p);
4441 }
4442
4443 static inline void
4444 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4445 {
4446         struct thread *t = &p->threads[p->thread_id];
4447         struct instruction *ip = t->ip;
4448
4449         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4450
4451         /* Structs. */
4452         ALU_HI(t, ip, <<);
4453
4454         /* Thread. */
4455         thread_ip_inc(p);
4456 }
4457
4458 static inline void
4459 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4460 {
4461         struct thread *t = &p->threads[p->thread_id];
4462         struct instruction *ip = t->ip;
4463
4464         TRACE("[Thread %2u] shr\n", p->thread_id);
4465
4466         /* Structs. */
4467         ALU(t, ip, >>);
4468
4469         /* Thread. */
4470         thread_ip_inc(p);
4471 }
4472
4473 static inline void
4474 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4475 {
4476         struct thread *t = &p->threads[p->thread_id];
4477         struct instruction *ip = t->ip;
4478
4479         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4480
4481         /* Structs. */
4482         ALU_MH(t, ip, >>);
4483
4484         /* Thread. */
4485         thread_ip_inc(p);
4486 }
4487
4488 static inline void
4489 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4490 {
4491         struct thread *t = &p->threads[p->thread_id];
4492         struct instruction *ip = t->ip;
4493
4494         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4495
4496         /* Structs. */
4497         ALU_HM(t, ip, >>);
4498
4499         /* Thread. */
4500         thread_ip_inc(p);
4501 }
4502
4503 static inline void
4504 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4505 {
4506         struct thread *t = &p->threads[p->thread_id];
4507         struct instruction *ip = t->ip;
4508
4509         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4510
4511         /* Structs. */
4512         ALU_HH(t, ip, >>);
4513
4514         /* Thread. */
4515         thread_ip_inc(p);
4516 }
4517
4518 static inline void
4519 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4520 {
4521         struct thread *t = &p->threads[p->thread_id];
4522         struct instruction *ip = t->ip;
4523
4524         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4525
4526         /* Structs. */
4527         ALU_MI(t, ip, >>);
4528
4529         /* Thread. */
4530         thread_ip_inc(p);
4531 }
4532
4533 static inline void
4534 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4535 {
4536         struct thread *t = &p->threads[p->thread_id];
4537         struct instruction *ip = t->ip;
4538
4539         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4540
4541         /* Structs. */
4542         ALU_HI(t, ip, >>);
4543
4544         /* Thread. */
4545         thread_ip_inc(p);
4546 }
4547
4548 static inline void
4549 instr_alu_and_exec(struct rte_swx_pipeline *p)
4550 {
4551         struct thread *t = &p->threads[p->thread_id];
4552         struct instruction *ip = t->ip;
4553
4554         TRACE("[Thread %2u] and\n", p->thread_id);
4555
4556         /* Structs. */
4557         ALU(t, ip, &);
4558
4559         /* Thread. */
4560         thread_ip_inc(p);
4561 }
4562
4563 static inline void
4564 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
4565 {
4566         struct thread *t = &p->threads[p->thread_id];
4567         struct instruction *ip = t->ip;
4568
4569         TRACE("[Thread %2u] and (s)\n", p->thread_id);
4570
4571         /* Structs. */
4572         ALU_S(t, ip, &);
4573
4574         /* Thread. */
4575         thread_ip_inc(p);
4576 }
4577
4578 static inline void
4579 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4580 {
4581         struct thread *t = &p->threads[p->thread_id];
4582         struct instruction *ip = t->ip;
4583
4584         TRACE("[Thread %2u] and (i)\n", p->thread_id);
4585
4586         /* Structs. */
4587         ALU_I(t, ip, &);
4588
4589         /* Thread. */
4590         thread_ip_inc(p);
4591 }
4592
4593 static inline void
4594 instr_alu_or_exec(struct rte_swx_pipeline *p)
4595 {
4596         struct thread *t = &p->threads[p->thread_id];
4597         struct instruction *ip = t->ip;
4598
4599         TRACE("[Thread %2u] or\n", p->thread_id);
4600
4601         /* Structs. */
4602         ALU(t, ip, |);
4603
4604         /* Thread. */
4605         thread_ip_inc(p);
4606 }
4607
4608 static inline void
4609 instr_alu_or_s_exec(struct rte_swx_pipeline *p)
4610 {
4611         struct thread *t = &p->threads[p->thread_id];
4612         struct instruction *ip = t->ip;
4613
4614         TRACE("[Thread %2u] or (s)\n", p->thread_id);
4615
4616         /* Structs. */
4617         ALU_S(t, ip, |);
4618
4619         /* Thread. */
4620         thread_ip_inc(p);
4621 }
4622
4623 static inline void
4624 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4625 {
4626         struct thread *t = &p->threads[p->thread_id];
4627         struct instruction *ip = t->ip;
4628
4629         TRACE("[Thread %2u] or (i)\n", p->thread_id);
4630
4631         /* Structs. */
4632         ALU_I(t, ip, |);
4633
4634         /* Thread. */
4635         thread_ip_inc(p);
4636 }
4637
4638 static inline void
4639 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4640 {
4641         struct thread *t = &p->threads[p->thread_id];
4642         struct instruction *ip = t->ip;
4643
4644         TRACE("[Thread %2u] xor\n", p->thread_id);
4645
4646         /* Structs. */
4647         ALU(t, ip, ^);
4648
4649         /* Thread. */
4650         thread_ip_inc(p);
4651 }
4652
4653 static inline void
4654 instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
4655 {
4656         struct thread *t = &p->threads[p->thread_id];
4657         struct instruction *ip = t->ip;
4658
4659         TRACE("[Thread %2u] xor (s)\n", p->thread_id);
4660
4661         /* Structs. */
4662         ALU_S(t, ip, ^);
4663
4664         /* Thread. */
4665         thread_ip_inc(p);
4666 }
4667
4668 static inline void
4669 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4670 {
4671         struct thread *t = &p->threads[p->thread_id];
4672         struct instruction *ip = t->ip;
4673
4674         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
4675
4676         /* Structs. */
4677         ALU_I(t, ip, ^);
4678
4679         /* Thread. */
4680         thread_ip_inc(p);
4681 }
4682
4683 static inline void
4684 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4685 {
4686         struct thread *t = &p->threads[p->thread_id];
4687         struct instruction *ip = t->ip;
4688         uint8_t *dst_struct, *src_struct;
4689         uint16_t *dst16_ptr, dst;
4690         uint64_t *src64_ptr, src64, src64_mask, src;
4691         uint64_t r;
4692
4693         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
4694
4695         /* Structs. */
4696         dst_struct = t->structs[ip->alu.dst.struct_id];
4697         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4698         dst = *dst16_ptr;
4699
4700         src_struct = t->structs[ip->alu.src.struct_id];
4701         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4702         src64 = *src64_ptr;
4703         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4704         src = src64 & src64_mask;
4705
4706         r = dst;
4707         r = ~r & 0xFFFF;
4708
4709         /* The first input (r) is a 16-bit number. The second and the third
4710          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
4711          * three numbers (output r) is a 34-bit number.
4712          */
4713         r += (src >> 32) + (src & 0xFFFFFFFF);
4714
4715         /* The first input is a 16-bit number. The second input is an 18-bit
4716          * number. In the worst case scenario, the sum of the two numbers is a
4717          * 19-bit number.
4718          */
4719         r = (r & 0xFFFF) + (r >> 16);
4720
4721         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4722          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
4723          */
4724         r = (r & 0xFFFF) + (r >> 16);
4725
4726         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4727          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4728          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
4729          * therefore the output r is always a 16-bit number.
4730          */
4731         r = (r & 0xFFFF) + (r >> 16);
4732
4733         r = ~r & 0xFFFF;
4734         r = r ? r : 0xFFFF;
4735
4736         *dst16_ptr = (uint16_t)r;
4737
4738         /* Thread. */
4739         thread_ip_inc(p);
4740 }
4741
4742 static inline void
4743 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4744 {
4745         struct thread *t = &p->threads[p->thread_id];
4746         struct instruction *ip = t->ip;
4747         uint8_t *dst_struct, *src_struct;
4748         uint16_t *dst16_ptr, dst;
4749         uint64_t *src64_ptr, src64, src64_mask, src;
4750         uint64_t r;
4751
4752         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
4753
4754         /* Structs. */
4755         dst_struct = t->structs[ip->alu.dst.struct_id];
4756         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4757         dst = *dst16_ptr;
4758
4759         src_struct = t->structs[ip->alu.src.struct_id];
4760         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4761         src64 = *src64_ptr;
4762         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4763         src = src64 & src64_mask;
4764
4765         r = dst;
4766         r = ~r & 0xFFFF;
4767
4768         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
4769          * the following sequence of operations in 2's complement arithmetic:
4770          *    a '- b = (a - b) % 0xFFFF.
4771          *
4772          * In order to prevent an underflow for the below subtraction, in which
4773          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
4774          * minuend), we first add a multiple of the 0xFFFF modulus to the
4775          * minuend. The number we add to the minuend needs to be a 34-bit number
4776          * or higher, so for readability reasons we picked the 36-bit multiple.
4777          * We are effectively turning the 16-bit minuend into a 36-bit number:
4778          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
4779          */
4780         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
4781
4782         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
4783          * result (the output r) is a 36-bit number.
4784          */
4785         r -= (src >> 32) + (src & 0xFFFFFFFF);
4786
4787         /* The first input is a 16-bit number. The second input is a 20-bit
4788          * number. Their sum is a 21-bit number.
4789          */
4790         r = (r & 0xFFFF) + (r >> 16);
4791
4792         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4793          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
4794          */
4795         r = (r & 0xFFFF) + (r >> 16);
4796
4797         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4798          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4799          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4800          * generated, therefore the output r is always a 16-bit number.
4801          */
4802         r = (r & 0xFFFF) + (r >> 16);
4803
4804         r = ~r & 0xFFFF;
4805         r = r ? r : 0xFFFF;
4806
4807         *dst16_ptr = (uint16_t)r;
4808
4809         /* Thread. */
4810         thread_ip_inc(p);
4811 }
4812
4813 static inline void
4814 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4815 {
4816         struct thread *t = &p->threads[p->thread_id];
4817         struct instruction *ip = t->ip;
4818         uint8_t *dst_struct, *src_struct;
4819         uint16_t *dst16_ptr;
4820         uint32_t *src32_ptr;
4821         uint64_t r0, r1;
4822
4823         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
4824
4825         /* Structs. */
4826         dst_struct = t->structs[ip->alu.dst.struct_id];
4827         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4828
4829         src_struct = t->structs[ip->alu.src.struct_id];
4830         src32_ptr = (uint32_t *)&src_struct[0];
4831
4832         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
4833         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
4834         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
4835         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
4836         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
4837
4838         /* The first input is a 16-bit number. The second input is a 19-bit
4839          * number. Their sum is a 20-bit number.
4840          */
4841         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4842
4843         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4844          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
4845          */
4846         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4847
4848         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4849          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4850          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
4851          * generated, therefore the output r is always a 16-bit number.
4852          */
4853         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4854
4855         r0 = ~r0 & 0xFFFF;
4856         r0 = r0 ? r0 : 0xFFFF;
4857
4858         *dst16_ptr = (uint16_t)r0;
4859
4860         /* Thread. */
4861         thread_ip_inc(p);
4862 }
4863
4864 static inline void
4865 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4866 {
4867         struct thread *t = &p->threads[p->thread_id];
4868         struct instruction *ip = t->ip;
4869         uint8_t *dst_struct, *src_struct;
4870         uint16_t *dst16_ptr;
4871         uint32_t *src32_ptr;
4872         uint64_t r = 0;
4873         uint32_t i;
4874
4875         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
4876
4877         /* Structs. */
4878         dst_struct = t->structs[ip->alu.dst.struct_id];
4879         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4880
4881         src_struct = t->structs[ip->alu.src.struct_id];
4882         src32_ptr = (uint32_t *)&src_struct[0];
4883
4884         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
4885          * Therefore, in the worst case scenario, a 35-bit number is added to a
4886          * 16-bit number (the input r), so the output r is 36-bit number.
4887          */
4888         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
4889                 r += *src32_ptr;
4890
4891         /* The first input is a 16-bit number. The second input is a 20-bit
4892          * number. Their sum is a 21-bit number.
4893          */
4894         r = (r & 0xFFFF) + (r >> 16);
4895
4896         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4897          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
4898          */
4899         r = (r & 0xFFFF) + (r >> 16);
4900
4901         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4902          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4903          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4904          * generated, therefore the output r is always a 16-bit number.
4905          */
4906         r = (r & 0xFFFF) + (r >> 16);
4907
4908         r = ~r & 0xFFFF;
4909         r = r ? r : 0xFFFF;
4910
4911         *dst16_ptr = (uint16_t)r;
4912
4913         /* Thread. */
4914         thread_ip_inc(p);
4915 }
4916
4917 /*
4918  * Register array.
4919  */
4920 static struct regarray *
4921 regarray_find(struct rte_swx_pipeline *p, const char *name);
4922
4923 static int
4924 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4925                       struct action *action,
4926                       char **tokens,
4927                       int n_tokens,
4928                       struct instruction *instr,
4929                       struct instruction_data *data __rte_unused)
4930 {
4931         char *regarray = tokens[1], *idx = tokens[2];
4932         struct regarray *r;
4933         struct field *fidx;
4934         uint32_t idx_struct_id, idx_val;
4935
4936         CHECK(n_tokens == 3, EINVAL);
4937
4938         r = regarray_find(p, regarray);
4939         CHECK(r, EINVAL);
4940
4941         /* REGPREFETCH_RH, REGPREFETCH_RM. */
4942         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4943         if (fidx) {
4944                 instr->type = INSTR_REGPREFETCH_RM;
4945                 if (idx[0] == 'h')
4946                         instr->type = INSTR_REGPREFETCH_RH;
4947
4948                 instr->regarray.regarray_id = r->id;
4949                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4950                 instr->regarray.idx.n_bits = fidx->n_bits;
4951                 instr->regarray.idx.offset = fidx->offset / 8;
4952                 instr->regarray.dstsrc_val = 0; /* Unused. */
4953                 return 0;
4954         }
4955
4956         /* REGPREFETCH_RI. */
4957         idx_val = strtoul(idx, &idx, 0);
4958         CHECK(!idx[0], EINVAL);
4959
4960         instr->type = INSTR_REGPREFETCH_RI;
4961         instr->regarray.regarray_id = r->id;
4962         instr->regarray.idx_val = idx_val;
4963         instr->regarray.dstsrc_val = 0; /* Unused. */
4964         return 0;
4965 }
4966
4967 static int
4968 instr_regrd_translate(struct rte_swx_pipeline *p,
4969                       struct action *action,
4970                       char **tokens,
4971                       int n_tokens,
4972                       struct instruction *instr,
4973                       struct instruction_data *data __rte_unused)
4974 {
4975         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4976         struct regarray *r;
4977         struct field *fdst, *fidx;
4978         uint32_t dst_struct_id, idx_struct_id, idx_val;
4979
4980         CHECK(n_tokens == 4, EINVAL);
4981
4982         r = regarray_find(p, regarray);
4983         CHECK(r, EINVAL);
4984
4985         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4986         CHECK(fdst, EINVAL);
4987
4988         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4989         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4990         if (fidx) {
4991                 instr->type = INSTR_REGRD_MRM;
4992                 if (dst[0] == 'h' && idx[0] != 'h')
4993                         instr->type = INSTR_REGRD_HRM;
4994                 if (dst[0] != 'h' && idx[0] == 'h')
4995                         instr->type = INSTR_REGRD_MRH;
4996                 if (dst[0] == 'h' && idx[0] == 'h')
4997                         instr->type = INSTR_REGRD_HRH;
4998
4999                 instr->regarray.regarray_id = r->id;
5000                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5001                 instr->regarray.idx.n_bits = fidx->n_bits;
5002                 instr->regarray.idx.offset = fidx->offset / 8;
5003                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5004                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
5005                 instr->regarray.dstsrc.offset = fdst->offset / 8;
5006                 return 0;
5007         }
5008
5009         /* REGRD_MRI, REGRD_HRI. */
5010         idx_val = strtoul(idx, &idx, 0);
5011         CHECK(!idx[0], EINVAL);
5012
5013         instr->type = INSTR_REGRD_MRI;
5014         if (dst[0] == 'h')
5015                 instr->type = INSTR_REGRD_HRI;
5016
5017         instr->regarray.regarray_id = r->id;
5018         instr->regarray.idx_val = idx_val;
5019         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5020         instr->regarray.dstsrc.n_bits = fdst->n_bits;
5021         instr->regarray.dstsrc.offset = fdst->offset / 8;
5022         return 0;
5023 }
5024
5025 static int
5026 instr_regwr_translate(struct rte_swx_pipeline *p,
5027                       struct action *action,
5028                       char **tokens,
5029                       int n_tokens,
5030                       struct instruction *instr,
5031                       struct instruction_data *data __rte_unused)
5032 {
5033         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5034         struct regarray *r;
5035         struct field *fidx, *fsrc;
5036         uint64_t src_val;
5037         uint32_t idx_struct_id, idx_val, src_struct_id;
5038
5039         CHECK(n_tokens == 4, EINVAL);
5040
5041         r = regarray_find(p, regarray);
5042         CHECK(r, EINVAL);
5043
5044         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
5045         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5046         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5047         if (fidx && fsrc) {
5048                 instr->type = INSTR_REGWR_RMM;
5049                 if (idx[0] == 'h' && src[0] != 'h')
5050                         instr->type = INSTR_REGWR_RHM;
5051                 if (idx[0] != 'h' && src[0] == 'h')
5052                         instr->type = INSTR_REGWR_RMH;
5053                 if (idx[0] == 'h' && src[0] == 'h')
5054                         instr->type = INSTR_REGWR_RHH;
5055
5056                 instr->regarray.regarray_id = r->id;
5057                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5058                 instr->regarray.idx.n_bits = fidx->n_bits;
5059                 instr->regarray.idx.offset = fidx->offset / 8;
5060                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5061                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5062                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5063                 return 0;
5064         }
5065
5066         /* REGWR_RHI, REGWR_RMI. */
5067         if (fidx && !fsrc) {
5068                 src_val = strtoull(src, &src, 0);
5069                 CHECK(!src[0], EINVAL);
5070
5071                 instr->type = INSTR_REGWR_RMI;
5072                 if (idx[0] == 'h')
5073                         instr->type = INSTR_REGWR_RHI;
5074
5075                 instr->regarray.regarray_id = r->id;
5076                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5077                 instr->regarray.idx.n_bits = fidx->n_bits;
5078                 instr->regarray.idx.offset = fidx->offset / 8;
5079                 instr->regarray.dstsrc_val = src_val;
5080                 return 0;
5081         }
5082
5083         /* REGWR_RIH, REGWR_RIM. */
5084         if (!fidx && fsrc) {
5085                 idx_val = strtoul(idx, &idx, 0);
5086                 CHECK(!idx[0], EINVAL);
5087
5088                 instr->type = INSTR_REGWR_RIM;
5089                 if (src[0] == 'h')
5090                         instr->type = INSTR_REGWR_RIH;
5091
5092                 instr->regarray.regarray_id = r->id;
5093                 instr->regarray.idx_val = idx_val;
5094                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5095                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5096                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5097                 return 0;
5098         }
5099
5100         /* REGWR_RII. */
5101         src_val = strtoull(src, &src, 0);
5102         CHECK(!src[0], EINVAL);
5103
5104         idx_val = strtoul(idx, &idx, 0);
5105         CHECK(!idx[0], EINVAL);
5106
5107         instr->type = INSTR_REGWR_RII;
5108         instr->regarray.idx_val = idx_val;
5109         instr->regarray.dstsrc_val = src_val;
5110
5111         return 0;
5112 }
5113
5114 static int
5115 instr_regadd_translate(struct rte_swx_pipeline *p,
5116                        struct action *action,
5117                        char **tokens,
5118                        int n_tokens,
5119                        struct instruction *instr,
5120                        struct instruction_data *data __rte_unused)
5121 {
5122         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5123         struct regarray *r;
5124         struct field *fidx, *fsrc;
5125         uint64_t src_val;
5126         uint32_t idx_struct_id, idx_val, src_struct_id;
5127
5128         CHECK(n_tokens == 4, EINVAL);
5129
5130         r = regarray_find(p, regarray);
5131         CHECK(r, EINVAL);
5132
5133         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
5134         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5135         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5136         if (fidx && fsrc) {
5137                 instr->type = INSTR_REGADD_RMM;
5138                 if (idx[0] == 'h' && src[0] != 'h')
5139                         instr->type = INSTR_REGADD_RHM;
5140                 if (idx[0] != 'h' && src[0] == 'h')
5141                         instr->type = INSTR_REGADD_RMH;
5142                 if (idx[0] == 'h' && src[0] == 'h')
5143                         instr->type = INSTR_REGADD_RHH;
5144
5145                 instr->regarray.regarray_id = r->id;
5146                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5147                 instr->regarray.idx.n_bits = fidx->n_bits;
5148                 instr->regarray.idx.offset = fidx->offset / 8;
5149                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5150                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5151                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5152                 return 0;
5153         }
5154
5155         /* REGADD_RHI, REGADD_RMI. */
5156         if (fidx && !fsrc) {
5157                 src_val = strtoull(src, &src, 0);
5158                 CHECK(!src[0], EINVAL);
5159
5160                 instr->type = INSTR_REGADD_RMI;
5161                 if (idx[0] == 'h')
5162                         instr->type = INSTR_REGADD_RHI;
5163
5164                 instr->regarray.regarray_id = r->id;
5165                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5166                 instr->regarray.idx.n_bits = fidx->n_bits;
5167                 instr->regarray.idx.offset = fidx->offset / 8;
5168                 instr->regarray.dstsrc_val = src_val;
5169                 return 0;
5170         }
5171
5172         /* REGADD_RIH, REGADD_RIM. */
5173         if (!fidx && fsrc) {
5174                 idx_val = strtoul(idx, &idx, 0);
5175                 CHECK(!idx[0], EINVAL);
5176
5177                 instr->type = INSTR_REGADD_RIM;
5178                 if (src[0] == 'h')
5179                         instr->type = INSTR_REGADD_RIH;
5180
5181                 instr->regarray.regarray_id = r->id;
5182                 instr->regarray.idx_val = idx_val;
5183                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5184                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5185                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5186                 return 0;
5187         }
5188
5189         /* REGADD_RII. */
5190         src_val = strtoull(src, &src, 0);
5191         CHECK(!src[0], EINVAL);
5192
5193         idx_val = strtoul(idx, &idx, 0);
5194         CHECK(!idx[0], EINVAL);
5195
5196         instr->type = INSTR_REGADD_RII;
5197         instr->regarray.idx_val = idx_val;
5198         instr->regarray.dstsrc_val = src_val;
5199         return 0;
5200 }
5201
5202 static inline uint64_t *
5203 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
5204 {
5205         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5206         return r->regarray;
5207 }
5208
5209 static inline uint64_t
5210 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5211 {
5212         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5213
5214         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5215         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5216         uint64_t idx64 = *idx64_ptr;
5217         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
5218         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5219
5220         return idx;
5221 }
5222
5223 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5224
5225 static inline uint64_t
5226 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5227 {
5228         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5229
5230         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5231         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5232         uint64_t idx64 = *idx64_ptr;
5233         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
5234
5235         return idx;
5236 }
5237
5238 #else
5239
5240 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
5241
5242 #endif
5243
5244 static inline uint64_t
5245 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5246 {
5247         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5248
5249         uint64_t idx = ip->regarray.idx_val & r->size_mask;
5250
5251         return idx;
5252 }
5253
5254 static inline uint64_t
5255 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
5256 {
5257         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5258         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5259         uint64_t src64 = *src64_ptr;
5260         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5261         uint64_t src = src64 & src64_mask;
5262
5263         return src;
5264 }
5265
5266 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5267
5268 static inline uint64_t
5269 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
5270 {
5271         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5272         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5273         uint64_t src64 = *src64_ptr;
5274         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
5275
5276         return src;
5277 }
5278
5279 #else
5280
5281 #define instr_regarray_src_nbo instr_regarray_src_hbo
5282
5283 #endif
5284
5285 static inline void
5286 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5287 {
5288         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5289         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5290         uint64_t dst64 = *dst64_ptr;
5291         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5292
5293         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5294
5295 }
5296
5297 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5298
5299 static inline void
5300 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5301 {
5302         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5303         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5304         uint64_t dst64 = *dst64_ptr;
5305         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5306
5307         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
5308         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5309 }
5310
5311 #else
5312
5313 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
5314
5315 #endif
5316
5317 static inline void
5318 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
5319 {
5320         struct thread *t = &p->threads[p->thread_id];
5321         struct instruction *ip = t->ip;
5322         uint64_t *regarray, idx;
5323
5324         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
5325
5326         /* Structs. */
5327         regarray = instr_regarray_regarray(p, ip);
5328         idx = instr_regarray_idx_nbo(p, t, ip);
5329         rte_prefetch0(&regarray[idx]);
5330
5331         /* Thread. */
5332         thread_ip_inc(p);
5333 }
5334
5335 static inline void
5336 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
5337 {
5338         struct thread *t = &p->threads[p->thread_id];
5339         struct instruction *ip = t->ip;
5340         uint64_t *regarray, idx;
5341
5342         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
5343
5344         /* Structs. */
5345         regarray = instr_regarray_regarray(p, ip);
5346         idx = instr_regarray_idx_hbo(p, t, ip);
5347         rte_prefetch0(&regarray[idx]);
5348
5349         /* Thread. */
5350         thread_ip_inc(p);
5351 }
5352
5353 static inline void
5354 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
5355 {
5356         struct thread *t = &p->threads[p->thread_id];
5357         struct instruction *ip = t->ip;
5358         uint64_t *regarray, idx;
5359
5360         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
5361
5362         /* Structs. */
5363         regarray = instr_regarray_regarray(p, ip);
5364         idx = instr_regarray_idx_imm(p, ip);
5365         rte_prefetch0(&regarray[idx]);
5366
5367         /* Thread. */
5368         thread_ip_inc(p);
5369 }
5370
5371 static inline void
5372 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
5373 {
5374         struct thread *t = &p->threads[p->thread_id];
5375         struct instruction *ip = t->ip;
5376         uint64_t *regarray, idx;
5377
5378         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
5379
5380         /* Structs. */
5381         regarray = instr_regarray_regarray(p, ip);
5382         idx = instr_regarray_idx_nbo(p, t, ip);
5383         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5384
5385         /* Thread. */
5386         thread_ip_inc(p);
5387 }
5388
5389 static inline void
5390 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
5391 {
5392         struct thread *t = &p->threads[p->thread_id];
5393         struct instruction *ip = t->ip;
5394         uint64_t *regarray, idx;
5395
5396         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
5397
5398         /* Structs. */
5399         regarray = instr_regarray_regarray(p, ip);
5400         idx = instr_regarray_idx_hbo(p, t, ip);
5401         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5402
5403         /* Thread. */
5404         thread_ip_inc(p);
5405 }
5406
5407 static inline void
5408 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
5409 {
5410         struct thread *t = &p->threads[p->thread_id];
5411         struct instruction *ip = t->ip;
5412         uint64_t *regarray, idx;
5413
5414         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
5415
5416         /* Structs. */
5417         regarray = instr_regarray_regarray(p, ip);
5418         idx = instr_regarray_idx_nbo(p, t, ip);
5419         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5420
5421         /* Thread. */
5422         thread_ip_inc(p);
5423 }
5424
5425 static inline void
5426 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
5427 {
5428         struct thread *t = &p->threads[p->thread_id];
5429         struct instruction *ip = t->ip;
5430         uint64_t *regarray, idx;
5431
5432         /* Structs. */
5433         regarray = instr_regarray_regarray(p, ip);
5434         idx = instr_regarray_idx_hbo(p, t, ip);
5435         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5436
5437         /* Thread. */
5438         thread_ip_inc(p);
5439 }
5440
5441 static inline void
5442 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
5443 {
5444         struct thread *t = &p->threads[p->thread_id];
5445         struct instruction *ip = t->ip;
5446         uint64_t *regarray, idx;
5447
5448         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
5449
5450         /* Structs. */
5451         regarray = instr_regarray_regarray(p, ip);
5452         idx = instr_regarray_idx_imm(p, ip);
5453         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5454
5455         /* Thread. */
5456         thread_ip_inc(p);
5457 }
5458
5459 static inline void
5460 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
5461 {
5462         struct thread *t = &p->threads[p->thread_id];
5463         struct instruction *ip = t->ip;
5464         uint64_t *regarray, idx;
5465
5466         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
5467
5468         /* Structs. */
5469         regarray = instr_regarray_regarray(p, ip);
5470         idx = instr_regarray_idx_imm(p, ip);
5471         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5472
5473         /* Thread. */
5474         thread_ip_inc(p);
5475 }
5476
5477 static inline void
5478 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
5479 {
5480         struct thread *t = &p->threads[p->thread_id];
5481         struct instruction *ip = t->ip;
5482         uint64_t *regarray, idx, src;
5483
5484         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
5485
5486         /* Structs. */
5487         regarray = instr_regarray_regarray(p, ip);
5488         idx = instr_regarray_idx_nbo(p, t, ip);
5489         src = instr_regarray_src_nbo(t, ip);
5490         regarray[idx] = src;
5491
5492         /* Thread. */
5493         thread_ip_inc(p);
5494 }
5495
5496 static inline void
5497 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
5498 {
5499         struct thread *t = &p->threads[p->thread_id];
5500         struct instruction *ip = t->ip;
5501         uint64_t *regarray, idx, src;
5502
5503         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
5504
5505         /* Structs. */
5506         regarray = instr_regarray_regarray(p, ip);
5507         idx = instr_regarray_idx_nbo(p, t, ip);
5508         src = instr_regarray_src_hbo(t, ip);
5509         regarray[idx] = src;
5510
5511         /* Thread. */
5512         thread_ip_inc(p);
5513 }
5514
5515 static inline void
5516 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
5517 {
5518         struct thread *t = &p->threads[p->thread_id];
5519         struct instruction *ip = t->ip;
5520         uint64_t *regarray, idx, src;
5521
5522         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
5523
5524         /* Structs. */
5525         regarray = instr_regarray_regarray(p, ip);
5526         idx = instr_regarray_idx_hbo(p, t, ip);
5527         src = instr_regarray_src_nbo(t, ip);
5528         regarray[idx] = src;
5529
5530         /* Thread. */
5531         thread_ip_inc(p);
5532 }
5533
5534 static inline void
5535 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
5536 {
5537         struct thread *t = &p->threads[p->thread_id];
5538         struct instruction *ip = t->ip;
5539         uint64_t *regarray, idx, src;
5540
5541         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
5542
5543         /* Structs. */
5544         regarray = instr_regarray_regarray(p, ip);
5545         idx = instr_regarray_idx_hbo(p, t, ip);
5546         src = instr_regarray_src_hbo(t, ip);
5547         regarray[idx] = src;
5548
5549         /* Thread. */
5550         thread_ip_inc(p);
5551 }
5552
5553 static inline void
5554 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
5555 {
5556         struct thread *t = &p->threads[p->thread_id];
5557         struct instruction *ip = t->ip;
5558         uint64_t *regarray, idx, src;
5559
5560         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
5561
5562         /* Structs. */
5563         regarray = instr_regarray_regarray(p, ip);
5564         idx = instr_regarray_idx_nbo(p, t, ip);
5565         src = ip->regarray.dstsrc_val;
5566         regarray[idx] = src;
5567
5568         /* Thread. */
5569         thread_ip_inc(p);
5570 }
5571
5572 static inline void
5573 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
5574 {
5575         struct thread *t = &p->threads[p->thread_id];
5576         struct instruction *ip = t->ip;
5577         uint64_t *regarray, idx, src;
5578
5579         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
5580
5581         /* Structs. */
5582         regarray = instr_regarray_regarray(p, ip);
5583         idx = instr_regarray_idx_hbo(p, t, ip);
5584         src = ip->regarray.dstsrc_val;
5585         regarray[idx] = src;
5586
5587         /* Thread. */
5588         thread_ip_inc(p);
5589 }
5590
5591 static inline void
5592 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
5593 {
5594         struct thread *t = &p->threads[p->thread_id];
5595         struct instruction *ip = t->ip;
5596         uint64_t *regarray, idx, src;
5597
5598         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
5599
5600         /* Structs. */
5601         regarray = instr_regarray_regarray(p, ip);
5602         idx = instr_regarray_idx_imm(p, ip);
5603         src = instr_regarray_src_nbo(t, ip);
5604         regarray[idx] = src;
5605
5606         /* Thread. */
5607         thread_ip_inc(p);
5608 }
5609
5610 static inline void
5611 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
5612 {
5613         struct thread *t = &p->threads[p->thread_id];
5614         struct instruction *ip = t->ip;
5615         uint64_t *regarray, idx, src;
5616
5617         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
5618
5619         /* Structs. */
5620         regarray = instr_regarray_regarray(p, ip);
5621         idx = instr_regarray_idx_imm(p, ip);
5622         src = instr_regarray_src_hbo(t, ip);
5623         regarray[idx] = src;
5624
5625         /* Thread. */
5626         thread_ip_inc(p);
5627 }
5628
5629 static inline void
5630 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
5631 {
5632         struct thread *t = &p->threads[p->thread_id];
5633         struct instruction *ip = t->ip;
5634         uint64_t *regarray, idx, src;
5635
5636         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
5637
5638         /* Structs. */
5639         regarray = instr_regarray_regarray(p, ip);
5640         idx = instr_regarray_idx_imm(p, ip);
5641         src = ip->regarray.dstsrc_val;
5642         regarray[idx] = src;
5643
5644         /* Thread. */
5645         thread_ip_inc(p);
5646 }
5647
5648 static inline void
5649 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
5650 {
5651         struct thread *t = &p->threads[p->thread_id];
5652         struct instruction *ip = t->ip;
5653         uint64_t *regarray, idx, src;
5654
5655         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
5656
5657         /* Structs. */
5658         regarray = instr_regarray_regarray(p, ip);
5659         idx = instr_regarray_idx_nbo(p, t, ip);
5660         src = instr_regarray_src_nbo(t, ip);
5661         regarray[idx] += src;
5662
5663         /* Thread. */
5664         thread_ip_inc(p);
5665 }
5666
5667 static inline void
5668 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
5669 {
5670         struct thread *t = &p->threads[p->thread_id];
5671         struct instruction *ip = t->ip;
5672         uint64_t *regarray, idx, src;
5673
5674         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
5675
5676         /* Structs. */
5677         regarray = instr_regarray_regarray(p, ip);
5678         idx = instr_regarray_idx_nbo(p, t, ip);
5679         src = instr_regarray_src_hbo(t, ip);
5680         regarray[idx] += src;
5681
5682         /* Thread. */
5683         thread_ip_inc(p);
5684 }
5685
5686 static inline void
5687 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
5688 {
5689         struct thread *t = &p->threads[p->thread_id];
5690         struct instruction *ip = t->ip;
5691         uint64_t *regarray, idx, src;
5692
5693         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
5694
5695         /* Structs. */
5696         regarray = instr_regarray_regarray(p, ip);
5697         idx = instr_regarray_idx_hbo(p, t, ip);
5698         src = instr_regarray_src_nbo(t, ip);
5699         regarray[idx] += src;
5700
5701         /* Thread. */
5702         thread_ip_inc(p);
5703 }
5704
5705 static inline void
5706 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
5707 {
5708         struct thread *t = &p->threads[p->thread_id];
5709         struct instruction *ip = t->ip;
5710         uint64_t *regarray, idx, src;
5711
5712         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
5713
5714         /* Structs. */
5715         regarray = instr_regarray_regarray(p, ip);
5716         idx = instr_regarray_idx_hbo(p, t, ip);
5717         src = instr_regarray_src_hbo(t, ip);
5718         regarray[idx] += src;
5719
5720         /* Thread. */
5721         thread_ip_inc(p);
5722 }
5723
5724 static inline void
5725 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
5726 {
5727         struct thread *t = &p->threads[p->thread_id];
5728         struct instruction *ip = t->ip;
5729         uint64_t *regarray, idx, src;
5730
5731         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
5732
5733         /* Structs. */
5734         regarray = instr_regarray_regarray(p, ip);
5735         idx = instr_regarray_idx_nbo(p, t, ip);
5736         src = ip->regarray.dstsrc_val;
5737         regarray[idx] += src;
5738
5739         /* Thread. */
5740         thread_ip_inc(p);
5741 }
5742
5743 static inline void
5744 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
5745 {
5746         struct thread *t = &p->threads[p->thread_id];
5747         struct instruction *ip = t->ip;
5748         uint64_t *regarray, idx, src;
5749
5750         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
5751
5752         /* Structs. */
5753         regarray = instr_regarray_regarray(p, ip);
5754         idx = instr_regarray_idx_hbo(p, t, ip);
5755         src = ip->regarray.dstsrc_val;
5756         regarray[idx] += src;
5757
5758         /* Thread. */
5759         thread_ip_inc(p);
5760 }
5761
5762 static inline void
5763 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
5764 {
5765         struct thread *t = &p->threads[p->thread_id];
5766         struct instruction *ip = t->ip;
5767         uint64_t *regarray, idx, src;
5768
5769         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
5770
5771         /* Structs. */
5772         regarray = instr_regarray_regarray(p, ip);
5773         idx = instr_regarray_idx_imm(p, ip);
5774         src = instr_regarray_src_nbo(t, ip);
5775         regarray[idx] += src;
5776
5777         /* Thread. */
5778         thread_ip_inc(p);
5779 }
5780
5781 static inline void
5782 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
5783 {
5784         struct thread *t = &p->threads[p->thread_id];
5785         struct instruction *ip = t->ip;
5786         uint64_t *regarray, idx, src;
5787
5788         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
5789
5790         /* Structs. */
5791         regarray = instr_regarray_regarray(p, ip);
5792         idx = instr_regarray_idx_imm(p, ip);
5793         src = instr_regarray_src_hbo(t, ip);
5794         regarray[idx] += src;
5795
5796         /* Thread. */
5797         thread_ip_inc(p);
5798 }
5799
5800 static inline void
5801 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
5802 {
5803         struct thread *t = &p->threads[p->thread_id];
5804         struct instruction *ip = t->ip;
5805         uint64_t *regarray, idx, src;
5806
5807         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
5808
5809         /* Structs. */
5810         regarray = instr_regarray_regarray(p, ip);
5811         idx = instr_regarray_idx_imm(p, ip);
5812         src = ip->regarray.dstsrc_val;
5813         regarray[idx] += src;
5814
5815         /* Thread. */
5816         thread_ip_inc(p);
5817 }
5818
5819 /*
5820  * metarray.
5821  */
5822 static struct metarray *
5823 metarray_find(struct rte_swx_pipeline *p, const char *name);
5824
5825 static int
5826 instr_metprefetch_translate(struct rte_swx_pipeline *p,
5827                             struct action *action,
5828                             char **tokens,
5829                             int n_tokens,
5830                             struct instruction *instr,
5831                             struct instruction_data *data __rte_unused)
5832 {
5833         char *metarray = tokens[1], *idx = tokens[2];
5834         struct metarray *m;
5835         struct field *fidx;
5836         uint32_t idx_struct_id, idx_val;
5837
5838         CHECK(n_tokens == 3, EINVAL);
5839
5840         m = metarray_find(p, metarray);
5841         CHECK(m, EINVAL);
5842
5843         /* METPREFETCH_H, METPREFETCH_M. */
5844         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5845         if (fidx) {
5846                 instr->type = INSTR_METPREFETCH_M;
5847                 if (idx[0] == 'h')
5848                         instr->type = INSTR_METPREFETCH_H;
5849
5850                 instr->meter.metarray_id = m->id;
5851                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5852                 instr->meter.idx.n_bits = fidx->n_bits;
5853                 instr->meter.idx.offset = fidx->offset / 8;
5854                 return 0;
5855         }
5856
5857         /* METPREFETCH_I. */
5858         idx_val = strtoul(idx, &idx, 0);
5859         CHECK(!idx[0], EINVAL);
5860
5861         instr->type = INSTR_METPREFETCH_I;
5862         instr->meter.metarray_id = m->id;
5863         instr->meter.idx_val = idx_val;
5864         return 0;
5865 }
5866
5867 static int
5868 instr_meter_translate(struct rte_swx_pipeline *p,
5869                       struct action *action,
5870                       char **tokens,
5871                       int n_tokens,
5872                       struct instruction *instr,
5873                       struct instruction_data *data __rte_unused)
5874 {
5875         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
5876         char *color_in = tokens[4], *color_out = tokens[5];
5877         struct metarray *m;
5878         struct field *fidx, *flength, *fcin, *fcout;
5879         uint32_t idx_struct_id, length_struct_id;
5880         uint32_t color_in_struct_id, color_out_struct_id;
5881
5882         CHECK(n_tokens == 6, EINVAL);
5883
5884         m = metarray_find(p, metarray);
5885         CHECK(m, EINVAL);
5886
5887         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5888
5889         flength = struct_field_parse(p, action, length, &length_struct_id);
5890         CHECK(flength, EINVAL);
5891
5892         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
5893
5894         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
5895         CHECK(fcout, EINVAL);
5896
5897         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
5898         if (fidx && fcin) {
5899                 instr->type = INSTR_METER_MMM;
5900                 if (idx[0] == 'h' && length[0] == 'h')
5901                         instr->type = INSTR_METER_HHM;
5902                 if (idx[0] == 'h' && length[0] != 'h')
5903                         instr->type = INSTR_METER_HMM;
5904                 if (idx[0] != 'h' && length[0] == 'h')
5905                         instr->type = INSTR_METER_MHM;
5906
5907                 instr->meter.metarray_id = m->id;
5908
5909                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5910                 instr->meter.idx.n_bits = fidx->n_bits;
5911                 instr->meter.idx.offset = fidx->offset / 8;
5912
5913                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5914                 instr->meter.length.n_bits = flength->n_bits;
5915                 instr->meter.length.offset = flength->offset / 8;
5916
5917                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5918                 instr->meter.color_in.n_bits = fcin->n_bits;
5919                 instr->meter.color_in.offset = fcin->offset / 8;
5920
5921                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5922                 instr->meter.color_out.n_bits = fcout->n_bits;
5923                 instr->meter.color_out.offset = fcout->offset / 8;
5924
5925                 return 0;
5926         }
5927
5928         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
5929         if (fidx && !fcin) {
5930                 uint32_t color_in_val = strtoul(color_in, &color_in, 0);
5931                 CHECK(!color_in[0], EINVAL);
5932
5933                 instr->type = INSTR_METER_MMI;
5934                 if (idx[0] == 'h' && length[0] == 'h')
5935                         instr->type = INSTR_METER_HHI;
5936                 if (idx[0] == 'h' && length[0] != 'h')
5937                         instr->type = INSTR_METER_HMI;
5938                 if (idx[0] != 'h' && length[0] == 'h')
5939                         instr->type = INSTR_METER_MHI;
5940
5941                 instr->meter.metarray_id = m->id;
5942
5943                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5944                 instr->meter.idx.n_bits = fidx->n_bits;
5945                 instr->meter.idx.offset = fidx->offset / 8;
5946
5947                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5948                 instr->meter.length.n_bits = flength->n_bits;
5949                 instr->meter.length.offset = flength->offset / 8;
5950
5951                 instr->meter.color_in_val = color_in_val;
5952
5953                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5954                 instr->meter.color_out.n_bits = fcout->n_bits;
5955                 instr->meter.color_out.offset = fcout->offset / 8;
5956
5957                 return 0;
5958         }
5959
5960         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5961         if (!fidx && fcin) {
5962                 uint32_t idx_val;
5963
5964                 idx_val = strtoul(idx, &idx, 0);
5965                 CHECK(!idx[0], EINVAL);
5966
5967                 instr->type = INSTR_METER_IMM;
5968                 if (length[0] == 'h')
5969                         instr->type = INSTR_METER_IHM;
5970
5971                 instr->meter.metarray_id = m->id;
5972
5973                 instr->meter.idx_val = idx_val;
5974
5975                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5976                 instr->meter.length.n_bits = flength->n_bits;
5977                 instr->meter.length.offset = flength->offset / 8;
5978
5979                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5980                 instr->meter.color_in.n_bits = fcin->n_bits;
5981                 instr->meter.color_in.offset = fcin->offset / 8;
5982
5983                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5984                 instr->meter.color_out.n_bits = fcout->n_bits;
5985                 instr->meter.color_out.offset = fcout->offset / 8;
5986
5987                 return 0;
5988         }
5989
5990         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5991         if (!fidx && !fcin) {
5992                 uint32_t idx_val, color_in_val;
5993
5994                 idx_val = strtoul(idx, &idx, 0);
5995                 CHECK(!idx[0], EINVAL);
5996
5997                 color_in_val = strtoul(color_in, &color_in, 0);
5998                 CHECK(!color_in[0], EINVAL);
5999
6000                 instr->type = INSTR_METER_IMI;
6001                 if (length[0] == 'h')
6002                         instr->type = INSTR_METER_IHI;
6003
6004                 instr->meter.metarray_id = m->id;
6005
6006                 instr->meter.idx_val = idx_val;
6007
6008                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6009                 instr->meter.length.n_bits = flength->n_bits;
6010                 instr->meter.length.offset = flength->offset / 8;
6011
6012                 instr->meter.color_in_val = color_in_val;
6013
6014                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6015                 instr->meter.color_out.n_bits = fcout->n_bits;
6016                 instr->meter.color_out.offset = fcout->offset / 8;
6017
6018                 return 0;
6019         }
6020
6021         CHECK(0, EINVAL);
6022 }
6023
6024 static inline struct meter *
6025 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6026 {
6027         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6028
6029         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6030         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6031         uint64_t idx64 = *idx64_ptr;
6032         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
6033         uint64_t idx = idx64 & idx64_mask & r->size_mask;
6034
6035         return &r->metarray[idx];
6036 }
6037
6038 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6039
6040 static inline struct meter *
6041 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6042 {
6043         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6044
6045         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6046         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6047         uint64_t idx64 = *idx64_ptr;
6048         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
6049
6050         return &r->metarray[idx];
6051 }
6052
6053 #else
6054
6055 #define instr_meter_idx_nbo instr_meter_idx_hbo
6056
6057 #endif
6058
6059 static inline struct meter *
6060 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
6061 {
6062         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6063
6064         uint64_t idx =  ip->meter.idx_val & r->size_mask;
6065
6066         return &r->metarray[idx];
6067 }
6068
6069 static inline uint32_t
6070 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
6071 {
6072         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6073         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6074         uint64_t src64 = *src64_ptr;
6075         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
6076         uint64_t src = src64 & src64_mask;
6077
6078         return (uint32_t)src;
6079 }
6080
6081 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6082
6083 static inline uint32_t
6084 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
6085 {
6086         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6087         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6088         uint64_t src64 = *src64_ptr;
6089         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
6090
6091         return (uint32_t)src;
6092 }
6093
6094 #else
6095
6096 #define instr_meter_length_nbo instr_meter_length_hbo
6097
6098 #endif
6099
6100 static inline enum rte_color
6101 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
6102 {
6103         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
6104         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
6105         uint64_t src64 = *src64_ptr;
6106         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
6107         uint64_t src = src64 & src64_mask;
6108
6109         return (enum rte_color)src;
6110 }
6111
6112 static inline void
6113 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
6114 {
6115         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
6116         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
6117         uint64_t dst64 = *dst64_ptr;
6118         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
6119
6120         uint64_t src = (uint64_t)color_out;
6121
6122         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6123 }
6124
6125 static inline void
6126 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
6127 {
6128         struct thread *t = &p->threads[p->thread_id];
6129         struct instruction *ip = t->ip;
6130         struct meter *m;
6131
6132         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
6133
6134         /* Structs. */
6135         m = instr_meter_idx_nbo(p, t, ip);
6136         rte_prefetch0(m);
6137
6138         /* Thread. */
6139         thread_ip_inc(p);
6140 }
6141
6142 static inline void
6143 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
6144 {
6145         struct thread *t = &p->threads[p->thread_id];
6146         struct instruction *ip = t->ip;
6147         struct meter *m;
6148
6149         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
6150
6151         /* Structs. */
6152         m = instr_meter_idx_hbo(p, t, ip);
6153         rte_prefetch0(m);
6154
6155         /* Thread. */
6156         thread_ip_inc(p);
6157 }
6158
6159 static inline void
6160 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
6161 {
6162         struct thread *t = &p->threads[p->thread_id];
6163         struct instruction *ip = t->ip;
6164         struct meter *m;
6165
6166         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
6167
6168         /* Structs. */
6169         m = instr_meter_idx_imm(p, ip);
6170         rte_prefetch0(m);
6171
6172         /* Thread. */
6173         thread_ip_inc(p);
6174 }
6175
6176 static inline void
6177 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
6178 {
6179         struct thread *t = &p->threads[p->thread_id];
6180         struct instruction *ip = t->ip;
6181         struct meter *m;
6182         uint64_t time, n_pkts, n_bytes;
6183         uint32_t length;
6184         enum rte_color color_in, color_out;
6185
6186         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
6187
6188         /* Structs. */
6189         m = instr_meter_idx_nbo(p, t, ip);
6190         rte_prefetch0(m->n_pkts);
6191         time = rte_get_tsc_cycles();
6192         length = instr_meter_length_nbo(t, ip);
6193         color_in = instr_meter_color_in_hbo(t, ip);
6194
6195         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6196                 &m->profile->profile,
6197                 time,
6198                 length,
6199                 color_in);
6200
6201         color_out &= m->color_mask;
6202
6203         n_pkts = m->n_pkts[color_out];
6204         n_bytes = m->n_bytes[color_out];
6205
6206         instr_meter_color_out_hbo_set(t, ip, color_out);
6207
6208         m->n_pkts[color_out] = n_pkts + 1;
6209         m->n_bytes[color_out] = n_bytes + length;
6210
6211         /* Thread. */
6212         thread_ip_inc(p);
6213 }
6214
6215 static inline void
6216 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
6217 {
6218         struct thread *t = &p->threads[p->thread_id];
6219         struct instruction *ip = t->ip;
6220         struct meter *m;
6221         uint64_t time, n_pkts, n_bytes;
6222         uint32_t length;
6223         enum rte_color color_in, color_out;
6224
6225         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
6226
6227         /* Structs. */
6228         m = instr_meter_idx_nbo(p, t, ip);
6229         rte_prefetch0(m->n_pkts);
6230         time = rte_get_tsc_cycles();
6231         length = instr_meter_length_nbo(t, ip);
6232         color_in = (enum rte_color)ip->meter.color_in_val;
6233
6234         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6235                 &m->profile->profile,
6236                 time,
6237                 length,
6238                 color_in);
6239
6240         color_out &= m->color_mask;
6241
6242         n_pkts = m->n_pkts[color_out];
6243         n_bytes = m->n_bytes[color_out];
6244
6245         instr_meter_color_out_hbo_set(t, ip, color_out);
6246
6247         m->n_pkts[color_out] = n_pkts + 1;
6248         m->n_bytes[color_out] = n_bytes + length;
6249
6250         /* Thread. */
6251         thread_ip_inc(p);
6252 }
6253
6254 static inline void
6255 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
6256 {
6257         struct thread *t = &p->threads[p->thread_id];
6258         struct instruction *ip = t->ip;
6259         struct meter *m;
6260         uint64_t time, n_pkts, n_bytes;
6261         uint32_t length;
6262         enum rte_color color_in, color_out;
6263
6264         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
6265
6266         /* Structs. */
6267         m = instr_meter_idx_nbo(p, t, ip);
6268         rte_prefetch0(m->n_pkts);
6269         time = rte_get_tsc_cycles();
6270         length = instr_meter_length_hbo(t, ip);
6271         color_in = instr_meter_color_in_hbo(t, ip);
6272
6273         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6274                 &m->profile->profile,
6275                 time,
6276                 length,
6277                 color_in);
6278
6279         color_out &= m->color_mask;
6280
6281         n_pkts = m->n_pkts[color_out];
6282         n_bytes = m->n_bytes[color_out];
6283
6284         instr_meter_color_out_hbo_set(t, ip, color_out);
6285
6286         m->n_pkts[color_out] = n_pkts + 1;
6287         m->n_bytes[color_out] = n_bytes + length;
6288
6289         /* Thread. */
6290         thread_ip_inc(p);
6291 }
6292 static inline void
6293 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
6294 {
6295         struct thread *t = &p->threads[p->thread_id];
6296         struct instruction *ip = t->ip;
6297         struct meter *m;
6298         uint64_t time, n_pkts, n_bytes;
6299         uint32_t length;
6300         enum rte_color color_in, color_out;
6301
6302         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
6303
6304         /* Structs. */
6305         m = instr_meter_idx_nbo(p, t, ip);
6306         rte_prefetch0(m->n_pkts);
6307         time = rte_get_tsc_cycles();
6308         length = instr_meter_length_hbo(t, ip);
6309         color_in = (enum rte_color)ip->meter.color_in_val;
6310
6311         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6312                 &m->profile->profile,
6313                 time,
6314                 length,
6315                 color_in);
6316
6317         color_out &= m->color_mask;
6318
6319         n_pkts = m->n_pkts[color_out];
6320         n_bytes = m->n_bytes[color_out];
6321
6322         instr_meter_color_out_hbo_set(t, ip, color_out);
6323
6324         m->n_pkts[color_out] = n_pkts + 1;
6325         m->n_bytes[color_out] = n_bytes + length;
6326
6327         /* Thread. */
6328         thread_ip_inc(p);
6329 }
6330
6331 static inline void
6332 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
6333 {
6334         struct thread *t = &p->threads[p->thread_id];
6335         struct instruction *ip = t->ip;
6336         struct meter *m;
6337         uint64_t time, n_pkts, n_bytes;
6338         uint32_t length;
6339         enum rte_color color_in, color_out;
6340
6341         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
6342
6343         /* Structs. */
6344         m = instr_meter_idx_hbo(p, t, ip);
6345         rte_prefetch0(m->n_pkts);
6346         time = rte_get_tsc_cycles();
6347         length = instr_meter_length_nbo(t, ip);
6348         color_in = instr_meter_color_in_hbo(t, ip);
6349
6350         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6351                 &m->profile->profile,
6352                 time,
6353                 length,
6354                 color_in);
6355
6356         color_out &= m->color_mask;
6357
6358         n_pkts = m->n_pkts[color_out];
6359         n_bytes = m->n_bytes[color_out];
6360
6361         instr_meter_color_out_hbo_set(t, ip, color_out);
6362
6363         m->n_pkts[color_out] = n_pkts + 1;
6364         m->n_bytes[color_out] = n_bytes + length;
6365
6366         /* Thread. */
6367         thread_ip_inc(p);
6368 }
6369
6370 static inline void
6371 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
6372 {
6373         struct thread *t = &p->threads[p->thread_id];
6374         struct instruction *ip = t->ip;
6375         struct meter *m;
6376         uint64_t time, n_pkts, n_bytes;
6377         uint32_t length;
6378         enum rte_color color_in, color_out;
6379
6380         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
6381
6382         /* Structs. */
6383         m = instr_meter_idx_hbo(p, t, ip);
6384         rte_prefetch0(m->n_pkts);
6385         time = rte_get_tsc_cycles();
6386         length = instr_meter_length_nbo(t, ip);
6387         color_in = (enum rte_color)ip->meter.color_in_val;
6388
6389         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6390                 &m->profile->profile,
6391                 time,
6392                 length,
6393                 color_in);
6394
6395         color_out &= m->color_mask;
6396
6397         n_pkts = m->n_pkts[color_out];
6398         n_bytes = m->n_bytes[color_out];
6399
6400         instr_meter_color_out_hbo_set(t, ip, color_out);
6401
6402         m->n_pkts[color_out] = n_pkts + 1;
6403         m->n_bytes[color_out] = n_bytes + length;
6404
6405         /* Thread. */
6406         thread_ip_inc(p);
6407 }
6408
6409 static inline void
6410 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
6411 {
6412         struct thread *t = &p->threads[p->thread_id];
6413         struct instruction *ip = t->ip;
6414         struct meter *m;
6415         uint64_t time, n_pkts, n_bytes;
6416         uint32_t length;
6417         enum rte_color color_in, color_out;
6418
6419         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
6420
6421         /* Structs. */
6422         m = instr_meter_idx_hbo(p, t, ip);
6423         rte_prefetch0(m->n_pkts);
6424         time = rte_get_tsc_cycles();
6425         length = instr_meter_length_hbo(t, ip);
6426         color_in = instr_meter_color_in_hbo(t, ip);
6427
6428         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6429                 &m->profile->profile,
6430                 time,
6431                 length,
6432                 color_in);
6433
6434         color_out &= m->color_mask;
6435
6436         n_pkts = m->n_pkts[color_out];
6437         n_bytes = m->n_bytes[color_out];
6438
6439         instr_meter_color_out_hbo_set(t, ip, color_out);
6440
6441         m->n_pkts[color_out] = n_pkts + 1;
6442         m->n_bytes[color_out] = n_bytes + length;
6443
6444         /* Thread. */
6445         thread_ip_inc(p);
6446 }
6447
6448 static inline void
6449 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
6450 {
6451         struct thread *t = &p->threads[p->thread_id];
6452         struct instruction *ip = t->ip;
6453         struct meter *m;
6454         uint64_t time, n_pkts, n_bytes;
6455         uint32_t length;
6456         enum rte_color color_in, color_out;
6457
6458         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
6459
6460         /* Structs. */
6461         m = instr_meter_idx_hbo(p, t, ip);
6462         rte_prefetch0(m->n_pkts);
6463         time = rte_get_tsc_cycles();
6464         length = instr_meter_length_hbo(t, ip);
6465         color_in = (enum rte_color)ip->meter.color_in_val;
6466
6467         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6468                 &m->profile->profile,
6469                 time,
6470                 length,
6471                 color_in);
6472
6473         color_out &= m->color_mask;
6474
6475         n_pkts = m->n_pkts[color_out];
6476         n_bytes = m->n_bytes[color_out];
6477
6478         instr_meter_color_out_hbo_set(t, ip, color_out);
6479
6480         m->n_pkts[color_out] = n_pkts + 1;
6481         m->n_bytes[color_out] = n_bytes + length;
6482
6483         /* Thread. */
6484         thread_ip_inc(p);
6485 }
6486
6487 static inline void
6488 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
6489 {
6490         struct thread *t = &p->threads[p->thread_id];
6491         struct instruction *ip = t->ip;
6492         struct meter *m;
6493         uint64_t time, n_pkts, n_bytes;
6494         uint32_t length;
6495         enum rte_color color_in, color_out;
6496
6497         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
6498
6499         /* Structs. */
6500         m = instr_meter_idx_imm(p, ip);
6501         rte_prefetch0(m->n_pkts);
6502         time = rte_get_tsc_cycles();
6503         length = instr_meter_length_nbo(t, ip);
6504         color_in = instr_meter_color_in_hbo(t, ip);
6505
6506         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6507                 &m->profile->profile,
6508                 time,
6509                 length,
6510                 color_in);
6511
6512         color_out &= m->color_mask;
6513
6514         n_pkts = m->n_pkts[color_out];
6515         n_bytes = m->n_bytes[color_out];
6516
6517         instr_meter_color_out_hbo_set(t, ip, color_out);
6518
6519         m->n_pkts[color_out] = n_pkts + 1;
6520         m->n_bytes[color_out] = n_bytes + length;
6521
6522         /* Thread. */
6523         thread_ip_inc(p);
6524 }
6525
6526 static inline void
6527 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
6528 {
6529         struct thread *t = &p->threads[p->thread_id];
6530         struct instruction *ip = t->ip;
6531         struct meter *m;
6532         uint64_t time, n_pkts, n_bytes;
6533         uint32_t length;
6534         enum rte_color color_in, color_out;
6535
6536         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
6537
6538         /* Structs. */
6539         m = instr_meter_idx_imm(p, ip);
6540         rte_prefetch0(m->n_pkts);
6541         time = rte_get_tsc_cycles();
6542         length = instr_meter_length_nbo(t, ip);
6543         color_in = (enum rte_color)ip->meter.color_in_val;
6544
6545         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6546                 &m->profile->profile,
6547                 time,
6548                 length,
6549                 color_in);
6550
6551         color_out &= m->color_mask;
6552
6553         n_pkts = m->n_pkts[color_out];
6554         n_bytes = m->n_bytes[color_out];
6555
6556         instr_meter_color_out_hbo_set(t, ip, color_out);
6557
6558         m->n_pkts[color_out] = n_pkts + 1;
6559         m->n_bytes[color_out] = n_bytes + length;
6560
6561         /* Thread. */
6562         thread_ip_inc(p);
6563 }
6564
6565 static inline void
6566 instr_meter_imm_exec(struct rte_swx_pipeline *p)
6567 {
6568         struct thread *t = &p->threads[p->thread_id];
6569         struct instruction *ip = t->ip;
6570         struct meter *m;
6571         uint64_t time, n_pkts, n_bytes;
6572         uint32_t length;
6573         enum rte_color color_in, color_out;
6574
6575         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
6576
6577         /* Structs. */
6578         m = instr_meter_idx_imm(p, ip);
6579         rte_prefetch0(m->n_pkts);
6580         time = rte_get_tsc_cycles();
6581         length = instr_meter_length_hbo(t, ip);
6582         color_in = instr_meter_color_in_hbo(t, ip);
6583
6584         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6585                 &m->profile->profile,
6586                 time,
6587                 length,
6588                 color_in);
6589
6590         color_out &= m->color_mask;
6591
6592         n_pkts = m->n_pkts[color_out];
6593         n_bytes = m->n_bytes[color_out];
6594
6595         instr_meter_color_out_hbo_set(t, ip, color_out);
6596
6597         m->n_pkts[color_out] = n_pkts + 1;
6598         m->n_bytes[color_out] = n_bytes + length;
6599
6600         /* Thread. */
6601         thread_ip_inc(p);
6602 }
6603 static inline void
6604 instr_meter_imi_exec(struct rte_swx_pipeline *p)
6605 {
6606         struct thread *t = &p->threads[p->thread_id];
6607         struct instruction *ip = t->ip;
6608         struct meter *m;
6609         uint64_t time, n_pkts, n_bytes;
6610         uint32_t length;
6611         enum rte_color color_in, color_out;
6612
6613         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
6614
6615         /* Structs. */
6616         m = instr_meter_idx_imm(p, ip);
6617         rte_prefetch0(m->n_pkts);
6618         time = rte_get_tsc_cycles();
6619         length = instr_meter_length_hbo(t, ip);
6620         color_in = (enum rte_color)ip->meter.color_in_val;
6621
6622         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6623                 &m->profile->profile,
6624                 time,
6625                 length,
6626                 color_in);
6627
6628         color_out &= m->color_mask;
6629
6630         n_pkts = m->n_pkts[color_out];
6631         n_bytes = m->n_bytes[color_out];
6632
6633         instr_meter_color_out_hbo_set(t, ip, color_out);
6634
6635         m->n_pkts[color_out] = n_pkts + 1;
6636         m->n_bytes[color_out] = n_bytes + length;
6637
6638         /* Thread. */
6639         thread_ip_inc(p);
6640 }
6641
6642 /*
6643  * jmp.
6644  */
6645 static struct action *
6646 action_find(struct rte_swx_pipeline *p, const char *name);
6647
6648 static int
6649 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
6650                     struct action *action __rte_unused,
6651                     char **tokens,
6652                     int n_tokens,
6653                     struct instruction *instr,
6654                     struct instruction_data *data)
6655 {
6656         CHECK(n_tokens == 2, EINVAL);
6657
6658         strcpy(data->jmp_label, tokens[1]);
6659
6660         instr->type = INSTR_JMP;
6661         instr->jmp.ip = NULL; /* Resolved later. */
6662         return 0;
6663 }
6664
6665 static int
6666 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
6667                           struct action *action __rte_unused,
6668                           char **tokens,
6669                           int n_tokens,
6670                           struct instruction *instr,
6671                           struct instruction_data *data)
6672 {
6673         struct header *h;
6674
6675         CHECK(n_tokens == 3, EINVAL);
6676
6677         strcpy(data->jmp_label, tokens[1]);
6678
6679         h = header_parse(p, tokens[2]);
6680         CHECK(h, EINVAL);
6681
6682         instr->type = INSTR_JMP_VALID;
6683         instr->jmp.ip = NULL; /* Resolved later. */
6684         instr->jmp.header_id = h->id;
6685         return 0;
6686 }
6687
6688 static int
6689 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
6690                             struct action *action __rte_unused,
6691                             char **tokens,
6692                             int n_tokens,
6693                             struct instruction *instr,
6694                             struct instruction_data *data)
6695 {
6696         struct header *h;
6697
6698         CHECK(n_tokens == 3, EINVAL);
6699
6700         strcpy(data->jmp_label, tokens[1]);
6701
6702         h = header_parse(p, tokens[2]);
6703         CHECK(h, EINVAL);
6704
6705         instr->type = INSTR_JMP_INVALID;
6706         instr->jmp.ip = NULL; /* Resolved later. */
6707         instr->jmp.header_id = h->id;
6708         return 0;
6709 }
6710
6711 static int
6712 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
6713                         struct action *action,
6714                         char **tokens,
6715                         int n_tokens,
6716                         struct instruction *instr,
6717                         struct instruction_data *data)
6718 {
6719         CHECK(!action, EINVAL);
6720         CHECK(n_tokens == 2, EINVAL);
6721
6722         strcpy(data->jmp_label, tokens[1]);
6723
6724         instr->type = INSTR_JMP_HIT;
6725         instr->jmp.ip = NULL; /* Resolved later. */
6726         return 0;
6727 }
6728
6729 static int
6730 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
6731                          struct action *action,
6732                          char **tokens,
6733                          int n_tokens,
6734                          struct instruction *instr,
6735                          struct instruction_data *data)
6736 {
6737         CHECK(!action, EINVAL);
6738         CHECK(n_tokens == 2, EINVAL);
6739
6740         strcpy(data->jmp_label, tokens[1]);
6741
6742         instr->type = INSTR_JMP_MISS;
6743         instr->jmp.ip = NULL; /* Resolved later. */
6744         return 0;
6745 }
6746
6747 static int
6748 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
6749                                struct action *action,
6750                                char **tokens,
6751                                int n_tokens,
6752                                struct instruction *instr,
6753                                struct instruction_data *data)
6754 {
6755         struct action *a;
6756
6757         CHECK(!action, EINVAL);
6758         CHECK(n_tokens == 3, EINVAL);
6759
6760         strcpy(data->jmp_label, tokens[1]);
6761
6762         a = action_find(p, tokens[2]);
6763         CHECK(a, EINVAL);
6764
6765         instr->type = INSTR_JMP_ACTION_HIT;
6766         instr->jmp.ip = NULL; /* Resolved later. */
6767         instr->jmp.action_id = a->id;
6768         return 0;
6769 }
6770
6771 static int
6772 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
6773                                 struct action *action,
6774                                 char **tokens,
6775                                 int n_tokens,
6776                                 struct instruction *instr,
6777                                 struct instruction_data *data)
6778 {
6779         struct action *a;
6780
6781         CHECK(!action, EINVAL);
6782         CHECK(n_tokens == 3, EINVAL);
6783
6784         strcpy(data->jmp_label, tokens[1]);
6785
6786         a = action_find(p, tokens[2]);
6787         CHECK(a, EINVAL);
6788
6789         instr->type = INSTR_JMP_ACTION_MISS;
6790         instr->jmp.ip = NULL; /* Resolved later. */
6791         instr->jmp.action_id = a->id;
6792         return 0;
6793 }
6794
6795 static int
6796 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
6797                        struct action *action,
6798                        char **tokens,
6799                        int n_tokens,
6800                        struct instruction *instr,
6801                        struct instruction_data *data)
6802 {
6803         char *a = tokens[2], *b = tokens[3];
6804         struct field *fa, *fb;
6805         uint64_t b_val;
6806         uint32_t a_struct_id, b_struct_id;
6807
6808         CHECK(n_tokens == 4, EINVAL);
6809
6810         strcpy(data->jmp_label, tokens[1]);
6811
6812         fa = struct_field_parse(p, action, a, &a_struct_id);
6813         CHECK(fa, EINVAL);
6814
6815         /* JMP_EQ or JMP_EQ_S. */
6816         fb = struct_field_parse(p, action, b, &b_struct_id);
6817         if (fb) {
6818                 instr->type = INSTR_JMP_EQ;
6819                 if ((a[0] == 'h' && b[0] != 'h') ||
6820                     (a[0] != 'h' && b[0] == 'h'))
6821                         instr->type = INSTR_JMP_EQ_S;
6822                 instr->jmp.ip = NULL; /* Resolved later. */
6823
6824                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6825                 instr->jmp.a.n_bits = fa->n_bits;
6826                 instr->jmp.a.offset = fa->offset / 8;
6827                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6828                 instr->jmp.b.n_bits = fb->n_bits;
6829                 instr->jmp.b.offset = fb->offset / 8;
6830                 return 0;
6831         }
6832
6833         /* JMP_EQ_I. */
6834         b_val = strtoull(b, &b, 0);
6835         CHECK(!b[0], EINVAL);
6836
6837         if (a[0] == 'h')
6838                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6839
6840         instr->type = INSTR_JMP_EQ_I;
6841         instr->jmp.ip = NULL; /* Resolved later. */
6842         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6843         instr->jmp.a.n_bits = fa->n_bits;
6844         instr->jmp.a.offset = fa->offset / 8;
6845         instr->jmp.b_val = b_val;
6846         return 0;
6847 }
6848
6849 static int
6850 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
6851                         struct action *action,
6852                         char **tokens,
6853                         int n_tokens,
6854                         struct instruction *instr,
6855                         struct instruction_data *data)
6856 {
6857         char *a = tokens[2], *b = tokens[3];
6858         struct field *fa, *fb;
6859         uint64_t b_val;
6860         uint32_t a_struct_id, b_struct_id;
6861
6862         CHECK(n_tokens == 4, EINVAL);
6863
6864         strcpy(data->jmp_label, tokens[1]);
6865
6866         fa = struct_field_parse(p, action, a, &a_struct_id);
6867         CHECK(fa, EINVAL);
6868
6869         /* JMP_NEQ or JMP_NEQ_S. */
6870         fb = struct_field_parse(p, action, b, &b_struct_id);
6871         if (fb) {
6872                 instr->type = INSTR_JMP_NEQ;
6873                 if ((a[0] == 'h' && b[0] != 'h') ||
6874                     (a[0] != 'h' && b[0] == 'h'))
6875                         instr->type = INSTR_JMP_NEQ_S;
6876                 instr->jmp.ip = NULL; /* Resolved later. */
6877
6878                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6879                 instr->jmp.a.n_bits = fa->n_bits;
6880                 instr->jmp.a.offset = fa->offset / 8;
6881                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6882                 instr->jmp.b.n_bits = fb->n_bits;
6883                 instr->jmp.b.offset = fb->offset / 8;
6884                 return 0;
6885         }
6886
6887         /* JMP_NEQ_I. */
6888         b_val = strtoull(b, &b, 0);
6889         CHECK(!b[0], EINVAL);
6890
6891         if (a[0] == 'h')
6892                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6893
6894         instr->type = INSTR_JMP_NEQ_I;
6895         instr->jmp.ip = NULL; /* Resolved later. */
6896         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6897         instr->jmp.a.n_bits = fa->n_bits;
6898         instr->jmp.a.offset = fa->offset / 8;
6899         instr->jmp.b_val = b_val;
6900         return 0;
6901 }
6902
6903 static int
6904 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
6905                        struct action *action,
6906                        char **tokens,
6907                        int n_tokens,
6908                        struct instruction *instr,
6909                        struct instruction_data *data)
6910 {
6911         char *a = tokens[2], *b = tokens[3];
6912         struct field *fa, *fb;
6913         uint64_t b_val;
6914         uint32_t a_struct_id, b_struct_id;
6915
6916         CHECK(n_tokens == 4, EINVAL);
6917
6918         strcpy(data->jmp_label, tokens[1]);
6919
6920         fa = struct_field_parse(p, action, a, &a_struct_id);
6921         CHECK(fa, EINVAL);
6922
6923         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
6924         fb = struct_field_parse(p, action, b, &b_struct_id);
6925         if (fb) {
6926                 instr->type = INSTR_JMP_LT;
6927                 if (a[0] == 'h' && b[0] != 'h')
6928                         instr->type = INSTR_JMP_LT_HM;
6929                 if (a[0] != 'h' && b[0] == 'h')
6930                         instr->type = INSTR_JMP_LT_MH;
6931                 if (a[0] == 'h' && b[0] == 'h')
6932                         instr->type = INSTR_JMP_LT_HH;
6933                 instr->jmp.ip = NULL; /* Resolved later. */
6934
6935                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6936                 instr->jmp.a.n_bits = fa->n_bits;
6937                 instr->jmp.a.offset = fa->offset / 8;
6938                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6939                 instr->jmp.b.n_bits = fb->n_bits;
6940                 instr->jmp.b.offset = fb->offset / 8;
6941                 return 0;
6942         }
6943
6944         /* JMP_LT_MI, JMP_LT_HI. */
6945         b_val = strtoull(b, &b, 0);
6946         CHECK(!b[0], EINVAL);
6947
6948         instr->type = INSTR_JMP_LT_MI;
6949         if (a[0] == 'h')
6950                 instr->type = INSTR_JMP_LT_HI;
6951         instr->jmp.ip = NULL; /* Resolved later. */
6952
6953         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6954         instr->jmp.a.n_bits = fa->n_bits;
6955         instr->jmp.a.offset = fa->offset / 8;
6956         instr->jmp.b_val = b_val;
6957         return 0;
6958 }
6959
6960 static int
6961 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
6962                        struct action *action,
6963                        char **tokens,
6964                        int n_tokens,
6965                        struct instruction *instr,
6966                        struct instruction_data *data)
6967 {
6968         char *a = tokens[2], *b = tokens[3];
6969         struct field *fa, *fb;
6970         uint64_t b_val;
6971         uint32_t a_struct_id, b_struct_id;
6972
6973         CHECK(n_tokens == 4, EINVAL);
6974
6975         strcpy(data->jmp_label, tokens[1]);
6976
6977         fa = struct_field_parse(p, action, a, &a_struct_id);
6978         CHECK(fa, EINVAL);
6979
6980         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
6981         fb = struct_field_parse(p, action, b, &b_struct_id);
6982         if (fb) {
6983                 instr->type = INSTR_JMP_GT;
6984                 if (a[0] == 'h' && b[0] != 'h')
6985                         instr->type = INSTR_JMP_GT_HM;
6986                 if (a[0] != 'h' && b[0] == 'h')
6987                         instr->type = INSTR_JMP_GT_MH;
6988                 if (a[0] == 'h' && b[0] == 'h')
6989                         instr->type = INSTR_JMP_GT_HH;
6990                 instr->jmp.ip = NULL; /* Resolved later. */
6991
6992                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6993                 instr->jmp.a.n_bits = fa->n_bits;
6994                 instr->jmp.a.offset = fa->offset / 8;
6995                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6996                 instr->jmp.b.n_bits = fb->n_bits;
6997                 instr->jmp.b.offset = fb->offset / 8;
6998                 return 0;
6999         }
7000
7001         /* JMP_GT_MI, JMP_GT_HI. */
7002         b_val = strtoull(b, &b, 0);
7003         CHECK(!b[0], EINVAL);
7004
7005         instr->type = INSTR_JMP_GT_MI;
7006         if (a[0] == 'h')
7007                 instr->type = INSTR_JMP_GT_HI;
7008         instr->jmp.ip = NULL; /* Resolved later. */
7009
7010         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7011         instr->jmp.a.n_bits = fa->n_bits;
7012         instr->jmp.a.offset = fa->offset / 8;
7013         instr->jmp.b_val = b_val;
7014         return 0;
7015 }
7016
7017 static inline void
7018 instr_jmp_exec(struct rte_swx_pipeline *p)
7019 {
7020         struct thread *t = &p->threads[p->thread_id];
7021         struct instruction *ip = t->ip;
7022
7023         TRACE("[Thread %2u] jmp\n", p->thread_id);
7024
7025         thread_ip_set(t, ip->jmp.ip);
7026 }
7027
7028 static inline void
7029 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
7030 {
7031         struct thread *t = &p->threads[p->thread_id];
7032         struct instruction *ip = t->ip;
7033         uint32_t header_id = ip->jmp.header_id;
7034
7035         TRACE("[Thread %2u] jmpv\n", p->thread_id);
7036
7037         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
7038 }
7039
7040 static inline void
7041 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
7042 {
7043         struct thread *t = &p->threads[p->thread_id];
7044         struct instruction *ip = t->ip;
7045         uint32_t header_id = ip->jmp.header_id;
7046
7047         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
7048
7049         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
7050 }
7051
7052 static inline void
7053 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
7054 {
7055         struct thread *t = &p->threads[p->thread_id];
7056         struct instruction *ip = t->ip;
7057         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
7058
7059         TRACE("[Thread %2u] jmph\n", p->thread_id);
7060
7061         t->ip = ip_next[t->hit];
7062 }
7063
7064 static inline void
7065 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
7066 {
7067         struct thread *t = &p->threads[p->thread_id];
7068         struct instruction *ip = t->ip;
7069         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
7070
7071         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
7072
7073         t->ip = ip_next[t->hit];
7074 }
7075
7076 static inline void
7077 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
7078 {
7079         struct thread *t = &p->threads[p->thread_id];
7080         struct instruction *ip = t->ip;
7081
7082         TRACE("[Thread %2u] jmpa\n", p->thread_id);
7083
7084         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
7085 }
7086
7087 static inline void
7088 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
7089 {
7090         struct thread *t = &p->threads[p->thread_id];
7091         struct instruction *ip = t->ip;
7092
7093         TRACE("[Thread %2u] jmpna\n", p->thread_id);
7094
7095         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
7096 }
7097
7098 static inline void
7099 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
7100 {
7101         struct thread *t = &p->threads[p->thread_id];
7102         struct instruction *ip = t->ip;
7103
7104         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
7105
7106         JMP_CMP(t, ip, ==);
7107 }
7108
7109 static inline void
7110 instr_jmp_eq_s_exec(struct rte_swx_pipeline *p)
7111 {
7112         struct thread *t = &p->threads[p->thread_id];
7113         struct instruction *ip = t->ip;
7114
7115         TRACE("[Thread %2u] jmpeq (s)\n", p->thread_id);
7116
7117         JMP_CMP_S(t, ip, ==);
7118 }
7119
7120 static inline void
7121 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
7122 {
7123         struct thread *t = &p->threads[p->thread_id];
7124         struct instruction *ip = t->ip;
7125
7126         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
7127
7128         JMP_CMP_I(t, ip, ==);
7129 }
7130
7131 static inline void
7132 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
7133 {
7134         struct thread *t = &p->threads[p->thread_id];
7135         struct instruction *ip = t->ip;
7136
7137         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
7138
7139         JMP_CMP(t, ip, !=);
7140 }
7141
7142 static inline void
7143 instr_jmp_neq_s_exec(struct rte_swx_pipeline *p)
7144 {
7145         struct thread *t = &p->threads[p->thread_id];
7146         struct instruction *ip = t->ip;
7147
7148         TRACE("[Thread %2u] jmpneq (s)\n", p->thread_id);
7149
7150         JMP_CMP_S(t, ip, !=);
7151 }
7152
7153 static inline void
7154 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
7155 {
7156         struct thread *t = &p->threads[p->thread_id];
7157         struct instruction *ip = t->ip;
7158
7159         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
7160
7161         JMP_CMP_I(t, ip, !=);
7162 }
7163
7164 static inline void
7165 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
7166 {
7167         struct thread *t = &p->threads[p->thread_id];
7168         struct instruction *ip = t->ip;
7169
7170         TRACE("[Thread %2u] jmplt\n", p->thread_id);
7171
7172         JMP_CMP(t, ip, <);
7173 }
7174
7175 static inline void
7176 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
7177 {
7178         struct thread *t = &p->threads[p->thread_id];
7179         struct instruction *ip = t->ip;
7180
7181         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
7182
7183         JMP_CMP_MH(t, ip, <);
7184 }
7185
7186 static inline void
7187 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
7188 {
7189         struct thread *t = &p->threads[p->thread_id];
7190         struct instruction *ip = t->ip;
7191
7192         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
7193
7194         JMP_CMP_HM(t, ip, <);
7195 }
7196
7197 static inline void
7198 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
7199 {
7200         struct thread *t = &p->threads[p->thread_id];
7201         struct instruction *ip = t->ip;
7202
7203         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
7204
7205         JMP_CMP_HH(t, ip, <);
7206 }
7207
7208 static inline void
7209 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
7210 {
7211         struct thread *t = &p->threads[p->thread_id];
7212         struct instruction *ip = t->ip;
7213
7214         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
7215
7216         JMP_CMP_MI(t, ip, <);
7217 }
7218
7219 static inline void
7220 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
7221 {
7222         struct thread *t = &p->threads[p->thread_id];
7223         struct instruction *ip = t->ip;
7224
7225         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
7226
7227         JMP_CMP_HI(t, ip, <);
7228 }
7229
7230 static inline void
7231 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
7232 {
7233         struct thread *t = &p->threads[p->thread_id];
7234         struct instruction *ip = t->ip;
7235
7236         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
7237
7238         JMP_CMP(t, ip, >);
7239 }
7240
7241 static inline void
7242 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
7243 {
7244         struct thread *t = &p->threads[p->thread_id];
7245         struct instruction *ip = t->ip;
7246
7247         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
7248
7249         JMP_CMP_MH(t, ip, >);
7250 }
7251
7252 static inline void
7253 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
7254 {
7255         struct thread *t = &p->threads[p->thread_id];
7256         struct instruction *ip = t->ip;
7257
7258         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
7259
7260         JMP_CMP_HM(t, ip, >);
7261 }
7262
7263 static inline void
7264 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
7265 {
7266         struct thread *t = &p->threads[p->thread_id];
7267         struct instruction *ip = t->ip;
7268
7269         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
7270
7271         JMP_CMP_HH(t, ip, >);
7272 }
7273
7274 static inline void
7275 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
7276 {
7277         struct thread *t = &p->threads[p->thread_id];
7278         struct instruction *ip = t->ip;
7279
7280         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
7281
7282         JMP_CMP_MI(t, ip, >);
7283 }
7284
7285 static inline void
7286 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
7287 {
7288         struct thread *t = &p->threads[p->thread_id];
7289         struct instruction *ip = t->ip;
7290
7291         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
7292
7293         JMP_CMP_HI(t, ip, >);
7294 }
7295
7296 /*
7297  * return.
7298  */
7299 static int
7300 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
7301                        struct action *action,
7302                        char **tokens __rte_unused,
7303                        int n_tokens,
7304                        struct instruction *instr,
7305                        struct instruction_data *data __rte_unused)
7306 {
7307         CHECK(action, EINVAL);
7308         CHECK(n_tokens == 1, EINVAL);
7309
7310         instr->type = INSTR_RETURN;
7311         return 0;
7312 }
7313
7314 static inline void
7315 instr_return_exec(struct rte_swx_pipeline *p)
7316 {
7317         struct thread *t = &p->threads[p->thread_id];
7318
7319         TRACE("[Thread %2u] return\n", p->thread_id);
7320
7321         t->ip = t->ret;
7322 }
7323
7324 static int
7325 instr_translate(struct rte_swx_pipeline *p,
7326                 struct action *action,
7327                 char *string,
7328                 struct instruction *instr,
7329                 struct instruction_data *data)
7330 {
7331         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
7332         int n_tokens = 0, tpos = 0;
7333
7334         /* Parse the instruction string into tokens. */
7335         for ( ; ; ) {
7336                 char *token;
7337
7338                 token = strtok_r(string, " \t\v", &string);
7339                 if (!token)
7340                         break;
7341
7342                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
7343                 CHECK_NAME(token, EINVAL);
7344
7345                 tokens[n_tokens] = token;
7346                 n_tokens++;
7347         }
7348
7349         CHECK(n_tokens, EINVAL);
7350
7351         /* Handle the optional instruction label. */
7352         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
7353                 strcpy(data->label, tokens[0]);
7354
7355                 tpos += 2;
7356                 CHECK(n_tokens - tpos, EINVAL);
7357         }
7358
7359         /* Identify the instruction type. */
7360         if (!strcmp(tokens[tpos], "rx"))
7361                 return instr_rx_translate(p,
7362                                           action,
7363                                           &tokens[tpos],
7364                                           n_tokens - tpos,
7365                                           instr,
7366                                           data);
7367
7368         if (!strcmp(tokens[tpos], "tx"))
7369                 return instr_tx_translate(p,
7370                                           action,
7371                                           &tokens[tpos],
7372                                           n_tokens - tpos,
7373                                           instr,
7374                                           data);
7375
7376         if (!strcmp(tokens[tpos], "drop"))
7377                 return instr_drop_translate(p,
7378                                             action,
7379                                             &tokens[tpos],
7380                                             n_tokens - tpos,
7381                                             instr,
7382                                             data);
7383
7384         if (!strcmp(tokens[tpos], "extract"))
7385                 return instr_hdr_extract_translate(p,
7386                                                    action,
7387                                                    &tokens[tpos],
7388                                                    n_tokens - tpos,
7389                                                    instr,
7390                                                    data);
7391
7392         if (!strcmp(tokens[tpos], "emit"))
7393                 return instr_hdr_emit_translate(p,
7394                                                 action,
7395                                                 &tokens[tpos],
7396                                                 n_tokens - tpos,
7397                                                 instr,
7398                                                 data);
7399
7400         if (!strcmp(tokens[tpos], "validate"))
7401                 return instr_hdr_validate_translate(p,
7402                                                     action,
7403                                                     &tokens[tpos],
7404                                                     n_tokens - tpos,
7405                                                     instr,
7406                                                     data);
7407
7408         if (!strcmp(tokens[tpos], "invalidate"))
7409                 return instr_hdr_invalidate_translate(p,
7410                                                       action,
7411                                                       &tokens[tpos],
7412                                                       n_tokens - tpos,
7413                                                       instr,
7414                                                       data);
7415
7416         if (!strcmp(tokens[tpos], "mov"))
7417                 return instr_mov_translate(p,
7418                                            action,
7419                                            &tokens[tpos],
7420                                            n_tokens - tpos,
7421                                            instr,
7422                                            data);
7423
7424         if (!strcmp(tokens[tpos], "dma"))
7425                 return instr_dma_translate(p,
7426                                            action,
7427                                            &tokens[tpos],
7428                                            n_tokens - tpos,
7429                                            instr,
7430                                            data);
7431
7432         if (!strcmp(tokens[tpos], "add"))
7433                 return instr_alu_add_translate(p,
7434                                                action,
7435                                                &tokens[tpos],
7436                                                n_tokens - tpos,
7437                                                instr,
7438                                                data);
7439
7440         if (!strcmp(tokens[tpos], "sub"))
7441                 return instr_alu_sub_translate(p,
7442                                                action,
7443                                                &tokens[tpos],
7444                                                n_tokens - tpos,
7445                                                instr,
7446                                                data);
7447
7448         if (!strcmp(tokens[tpos], "ckadd"))
7449                 return instr_alu_ckadd_translate(p,
7450                                                  action,
7451                                                  &tokens[tpos],
7452                                                  n_tokens - tpos,
7453                                                  instr,
7454                                                  data);
7455
7456         if (!strcmp(tokens[tpos], "cksub"))
7457                 return instr_alu_cksub_translate(p,
7458                                                  action,
7459                                                  &tokens[tpos],
7460                                                  n_tokens - tpos,
7461                                                  instr,
7462                                                  data);
7463
7464         if (!strcmp(tokens[tpos], "and"))
7465                 return instr_alu_and_translate(p,
7466                                                action,
7467                                                &tokens[tpos],
7468                                                n_tokens - tpos,
7469                                                instr,
7470                                                data);
7471
7472         if (!strcmp(tokens[tpos], "or"))
7473                 return instr_alu_or_translate(p,
7474                                               action,
7475                                               &tokens[tpos],
7476                                               n_tokens - tpos,
7477                                               instr,
7478                                               data);
7479
7480         if (!strcmp(tokens[tpos], "xor"))
7481                 return instr_alu_xor_translate(p,
7482                                                action,
7483                                                &tokens[tpos],
7484                                                n_tokens - tpos,
7485                                                instr,
7486                                                data);
7487
7488         if (!strcmp(tokens[tpos], "shl"))
7489                 return instr_alu_shl_translate(p,
7490                                                action,
7491                                                &tokens[tpos],
7492                                                n_tokens - tpos,
7493                                                instr,
7494                                                data);
7495
7496         if (!strcmp(tokens[tpos], "shr"))
7497                 return instr_alu_shr_translate(p,
7498                                                action,
7499                                                &tokens[tpos],
7500                                                n_tokens - tpos,
7501                                                instr,
7502                                                data);
7503
7504         if (!strcmp(tokens[tpos], "regprefetch"))
7505                 return instr_regprefetch_translate(p,
7506                                                    action,
7507                                                    &tokens[tpos],
7508                                                    n_tokens - tpos,
7509                                                    instr,
7510                                                    data);
7511
7512         if (!strcmp(tokens[tpos], "regrd"))
7513                 return instr_regrd_translate(p,
7514                                              action,
7515                                              &tokens[tpos],
7516                                              n_tokens - tpos,
7517                                              instr,
7518                                              data);
7519
7520         if (!strcmp(tokens[tpos], "regwr"))
7521                 return instr_regwr_translate(p,
7522                                              action,
7523                                              &tokens[tpos],
7524                                              n_tokens - tpos,
7525                                              instr,
7526                                              data);
7527
7528         if (!strcmp(tokens[tpos], "regadd"))
7529                 return instr_regadd_translate(p,
7530                                               action,
7531                                               &tokens[tpos],
7532                                               n_tokens - tpos,
7533                                               instr,
7534                                               data);
7535
7536         if (!strcmp(tokens[tpos], "metprefetch"))
7537                 return instr_metprefetch_translate(p,
7538                                                    action,
7539                                                    &tokens[tpos],
7540                                                    n_tokens - tpos,
7541                                                    instr,
7542                                                    data);
7543
7544         if (!strcmp(tokens[tpos], "meter"))
7545                 return instr_meter_translate(p,
7546                                              action,
7547                                              &tokens[tpos],
7548                                              n_tokens - tpos,
7549                                              instr,
7550                                              data);
7551
7552         if (!strcmp(tokens[tpos], "table"))
7553                 return instr_table_translate(p,
7554                                              action,
7555                                              &tokens[tpos],
7556                                              n_tokens - tpos,
7557                                              instr,
7558                                              data);
7559
7560         if (!strcmp(tokens[tpos], "extern"))
7561                 return instr_extern_translate(p,
7562                                               action,
7563                                               &tokens[tpos],
7564                                               n_tokens - tpos,
7565                                               instr,
7566                                               data);
7567
7568         if (!strcmp(tokens[tpos], "jmp"))
7569                 return instr_jmp_translate(p,
7570                                            action,
7571                                            &tokens[tpos],
7572                                            n_tokens - tpos,
7573                                            instr,
7574                                            data);
7575
7576         if (!strcmp(tokens[tpos], "jmpv"))
7577                 return instr_jmp_valid_translate(p,
7578                                                  action,
7579                                                  &tokens[tpos],
7580                                                  n_tokens - tpos,
7581                                                  instr,
7582                                                  data);
7583
7584         if (!strcmp(tokens[tpos], "jmpnv"))
7585                 return instr_jmp_invalid_translate(p,
7586                                                    action,
7587                                                    &tokens[tpos],
7588                                                    n_tokens - tpos,
7589                                                    instr,
7590                                                    data);
7591
7592         if (!strcmp(tokens[tpos], "jmph"))
7593                 return instr_jmp_hit_translate(p,
7594                                                action,
7595                                                &tokens[tpos],
7596                                                n_tokens - tpos,
7597                                                instr,
7598                                                data);
7599
7600         if (!strcmp(tokens[tpos], "jmpnh"))
7601                 return instr_jmp_miss_translate(p,
7602                                                 action,
7603                                                 &tokens[tpos],
7604                                                 n_tokens - tpos,
7605                                                 instr,
7606                                                 data);
7607
7608         if (!strcmp(tokens[tpos], "jmpa"))
7609                 return instr_jmp_action_hit_translate(p,
7610                                                       action,
7611                                                       &tokens[tpos],
7612                                                       n_tokens - tpos,
7613                                                       instr,
7614                                                       data);
7615
7616         if (!strcmp(tokens[tpos], "jmpna"))
7617                 return instr_jmp_action_miss_translate(p,
7618                                                        action,
7619                                                        &tokens[tpos],
7620                                                        n_tokens - tpos,
7621                                                        instr,
7622                                                        data);
7623
7624         if (!strcmp(tokens[tpos], "jmpeq"))
7625                 return instr_jmp_eq_translate(p,
7626                                               action,
7627                                               &tokens[tpos],
7628                                               n_tokens - tpos,
7629                                               instr,
7630                                               data);
7631
7632         if (!strcmp(tokens[tpos], "jmpneq"))
7633                 return instr_jmp_neq_translate(p,
7634                                                action,
7635                                                &tokens[tpos],
7636                                                n_tokens - tpos,
7637                                                instr,
7638                                                data);
7639
7640         if (!strcmp(tokens[tpos], "jmplt"))
7641                 return instr_jmp_lt_translate(p,
7642                                               action,
7643                                               &tokens[tpos],
7644                                               n_tokens - tpos,
7645                                               instr,
7646                                               data);
7647
7648         if (!strcmp(tokens[tpos], "jmpgt"))
7649                 return instr_jmp_gt_translate(p,
7650                                               action,
7651                                               &tokens[tpos],
7652                                               n_tokens - tpos,
7653                                               instr,
7654                                               data);
7655
7656         if (!strcmp(tokens[tpos], "return"))
7657                 return instr_return_translate(p,
7658                                               action,
7659                                               &tokens[tpos],
7660                                               n_tokens - tpos,
7661                                               instr,
7662                                               data);
7663
7664         CHECK(0, EINVAL);
7665 }
7666
7667 static struct instruction_data *
7668 label_find(struct instruction_data *data, uint32_t n, const char *label)
7669 {
7670         uint32_t i;
7671
7672         for (i = 0; i < n; i++)
7673                 if (!strcmp(label, data[i].label))
7674                         return &data[i];
7675
7676         return NULL;
7677 }
7678
7679 static uint32_t
7680 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
7681 {
7682         uint32_t count = 0, i;
7683
7684         if (!label[0])
7685                 return 0;
7686
7687         for (i = 0; i < n; i++)
7688                 if (!strcmp(label, data[i].jmp_label))
7689                         count++;
7690
7691         return count;
7692 }
7693
7694 static int
7695 instr_label_check(struct instruction_data *instruction_data,
7696                   uint32_t n_instructions)
7697 {
7698         uint32_t i;
7699
7700         /* Check that all instruction labels are unique. */
7701         for (i = 0; i < n_instructions; i++) {
7702                 struct instruction_data *data = &instruction_data[i];
7703                 char *label = data->label;
7704                 uint32_t j;
7705
7706                 if (!label[0])
7707                         continue;
7708
7709                 for (j = i + 1; j < n_instructions; j++)
7710                         CHECK(strcmp(label, data[j].label), EINVAL);
7711         }
7712
7713         /* Get users for each instruction label. */
7714         for (i = 0; i < n_instructions; i++) {
7715                 struct instruction_data *data = &instruction_data[i];
7716                 char *label = data->label;
7717
7718                 data->n_users = label_is_used(instruction_data,
7719                                               n_instructions,
7720                                               label);
7721         }
7722
7723         return 0;
7724 }
7725
7726 static int
7727 instr_jmp_resolve(struct instruction *instructions,
7728                   struct instruction_data *instruction_data,
7729                   uint32_t n_instructions)
7730 {
7731         uint32_t i;
7732
7733         for (i = 0; i < n_instructions; i++) {
7734                 struct instruction *instr = &instructions[i];
7735                 struct instruction_data *data = &instruction_data[i];
7736                 struct instruction_data *found;
7737
7738                 if (!instruction_is_jmp(instr))
7739                         continue;
7740
7741                 found = label_find(instruction_data,
7742                                    n_instructions,
7743                                    data->jmp_label);
7744                 CHECK(found, EINVAL);
7745
7746                 instr->jmp.ip = &instructions[found - instruction_data];
7747         }
7748
7749         return 0;
7750 }
7751
7752 static int
7753 instr_verify(struct rte_swx_pipeline *p __rte_unused,
7754              struct action *a,
7755              struct instruction *instr,
7756              struct instruction_data *data __rte_unused,
7757              uint32_t n_instructions)
7758 {
7759         if (!a) {
7760                 enum instruction_type type;
7761                 uint32_t i;
7762
7763                 /* Check that the first instruction is rx. */
7764                 CHECK(instr[0].type == INSTR_RX, EINVAL);
7765
7766                 /* Check that there is at least one tx instruction. */
7767                 for (i = 0; i < n_instructions; i++) {
7768                         type = instr[i].type;
7769
7770                         if (instruction_is_tx(type))
7771                                 break;
7772                 }
7773                 CHECK(i < n_instructions, EINVAL);
7774
7775                 /* Check that the last instruction is either tx or unconditional
7776                  * jump.
7777                  */
7778                 type = instr[n_instructions - 1].type;
7779                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
7780         }
7781
7782         if (a) {
7783                 enum instruction_type type;
7784                 uint32_t i;
7785
7786                 /* Check that there is at least one return or tx instruction. */
7787                 for (i = 0; i < n_instructions; i++) {
7788                         type = instr[i].type;
7789
7790                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
7791                                 break;
7792                 }
7793                 CHECK(i < n_instructions, EINVAL);
7794         }
7795
7796         return 0;
7797 }
7798
7799 static int
7800 instr_pattern_extract_many_detect(struct instruction *instr,
7801                                   struct instruction_data *data,
7802                                   uint32_t n_instr,
7803                                   uint32_t *n_pattern_instr)
7804 {
7805         uint32_t i;
7806
7807         for (i = 0; i < n_instr; i++) {
7808                 if (data[i].invalid)
7809                         break;
7810
7811                 if (instr[i].type != INSTR_HDR_EXTRACT)
7812                         break;
7813
7814                 if (i == RTE_DIM(instr->io.hdr.header_id))
7815                         break;
7816
7817                 if (i && data[i].n_users)
7818                         break;
7819         }
7820
7821         if (i < 2)
7822                 return 0;
7823
7824         *n_pattern_instr = i;
7825         return 1;
7826 }
7827
7828 static void
7829 instr_pattern_extract_many_optimize(struct instruction *instr,
7830                                     struct instruction_data *data,
7831                                     uint32_t n_instr)
7832 {
7833         uint32_t i;
7834
7835         for (i = 1; i < n_instr; i++) {
7836                 instr[0].type++;
7837                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7838                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7839                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7840
7841                 data[i].invalid = 1;
7842         }
7843 }
7844
7845 static int
7846 instr_pattern_emit_many_tx_detect(struct instruction *instr,
7847                                   struct instruction_data *data,
7848                                   uint32_t n_instr,
7849                                   uint32_t *n_pattern_instr)
7850 {
7851         uint32_t i;
7852
7853         for (i = 0; i < n_instr; i++) {
7854                 if (data[i].invalid)
7855                         break;
7856
7857                 if (instr[i].type != INSTR_HDR_EMIT)
7858                         break;
7859
7860                 if (i == RTE_DIM(instr->io.hdr.header_id))
7861                         break;
7862
7863                 if (i && data[i].n_users)
7864                         break;
7865         }
7866
7867         if (!i)
7868                 return 0;
7869
7870         if (!instruction_is_tx(instr[i].type))
7871                 return 0;
7872
7873         if (data[i].n_users)
7874                 return 0;
7875
7876         i++;
7877
7878         *n_pattern_instr = i;
7879         return 1;
7880 }
7881
7882 static void
7883 instr_pattern_emit_many_tx_optimize(struct instruction *instr,
7884                                     struct instruction_data *data,
7885                                     uint32_t n_instr)
7886 {
7887         uint32_t i;
7888
7889         /* Any emit instruction in addition to the first one. */
7890         for (i = 1; i < n_instr - 1; i++) {
7891                 instr[0].type++;
7892                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7893                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7894                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7895
7896                 data[i].invalid = 1;
7897         }
7898
7899         /* The TX instruction is the last one in the pattern. */
7900         instr[0].type++;
7901         instr[0].io.io.offset = instr[i].io.io.offset;
7902         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7903         data[i].invalid = 1;
7904 }
7905
7906 static int
7907 instr_pattern_dma_many_detect(struct instruction *instr,
7908                               struct instruction_data *data,
7909                               uint32_t n_instr,
7910                               uint32_t *n_pattern_instr)
7911 {
7912         uint32_t i;
7913
7914         for (i = 0; i < n_instr; i++) {
7915                 if (data[i].invalid)
7916                         break;
7917
7918                 if (instr[i].type != INSTR_DMA_HT)
7919                         break;
7920
7921                 if (i == RTE_DIM(instr->dma.dst.header_id))
7922                         break;
7923
7924                 if (i && data[i].n_users)
7925                         break;
7926         }
7927
7928         if (i < 2)
7929                 return 0;
7930
7931         *n_pattern_instr = i;
7932         return 1;
7933 }
7934
7935 static void
7936 instr_pattern_dma_many_optimize(struct instruction *instr,
7937                                 struct instruction_data *data,
7938                                 uint32_t n_instr)
7939 {
7940         uint32_t i;
7941
7942         for (i = 1; i < n_instr; i++) {
7943                 instr[0].type++;
7944                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7945                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7946                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7947                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7948
7949                 data[i].invalid = 1;
7950         }
7951 }
7952
7953 static uint32_t
7954 instr_optimize(struct instruction *instructions,
7955                struct instruction_data *instruction_data,
7956                uint32_t n_instructions)
7957 {
7958         uint32_t i, pos = 0;
7959
7960         for (i = 0; i < n_instructions; ) {
7961                 struct instruction *instr = &instructions[i];
7962                 struct instruction_data *data = &instruction_data[i];
7963                 uint32_t n_instr = 0;
7964                 int detected;
7965
7966                 /* Extract many. */
7967                 detected = instr_pattern_extract_many_detect(instr,
7968                                                              data,
7969                                                              n_instructions - i,
7970                                                              &n_instr);
7971                 if (detected) {
7972                         instr_pattern_extract_many_optimize(instr,
7973                                                             data,
7974                                                             n_instr);
7975                         i += n_instr;
7976                         continue;
7977                 }
7978
7979                 /* Emit many + TX. */
7980                 detected = instr_pattern_emit_many_tx_detect(instr,
7981                                                              data,
7982                                                              n_instructions - i,
7983                                                              &n_instr);
7984                 if (detected) {
7985                         instr_pattern_emit_many_tx_optimize(instr,
7986                                                             data,
7987                                                             n_instr);
7988                         i += n_instr;
7989                         continue;
7990                 }
7991
7992                 /* DMA many. */
7993                 detected = instr_pattern_dma_many_detect(instr,
7994                                                          data,
7995                                                          n_instructions - i,
7996                                                          &n_instr);
7997                 if (detected) {
7998                         instr_pattern_dma_many_optimize(instr, data, n_instr);
7999                         i += n_instr;
8000                         continue;
8001                 }
8002
8003                 /* No pattern starting at the current instruction. */
8004                 i++;
8005         }
8006
8007         /* Eliminate the invalid instructions that have been optimized out. */
8008         for (i = 0; i < n_instructions; i++) {
8009                 struct instruction *instr = &instructions[i];
8010                 struct instruction_data *data = &instruction_data[i];
8011
8012                 if (data->invalid)
8013                         continue;
8014
8015                 if (i != pos) {
8016                         memcpy(&instructions[pos], instr, sizeof(*instr));
8017                         memcpy(&instruction_data[pos], data, sizeof(*data));
8018                 }
8019
8020                 pos++;
8021         }
8022
8023         return pos;
8024 }
8025
8026 static int
8027 instruction_config(struct rte_swx_pipeline *p,
8028                    struct action *a,
8029                    const char **instructions,
8030                    uint32_t n_instructions)
8031 {
8032         struct instruction *instr = NULL;
8033         struct instruction_data *data = NULL;
8034         int err = 0;
8035         uint32_t i;
8036
8037         CHECK(n_instructions, EINVAL);
8038         CHECK(instructions, EINVAL);
8039         for (i = 0; i < n_instructions; i++)
8040                 CHECK_INSTRUCTION(instructions[i], EINVAL);
8041
8042         /* Memory allocation. */
8043         instr = calloc(n_instructions, sizeof(struct instruction));
8044         if (!instr) {
8045                 err = ENOMEM;
8046                 goto error;
8047         }
8048
8049         data = calloc(n_instructions, sizeof(struct instruction_data));
8050         if (!data) {
8051                 err = ENOMEM;
8052                 goto error;
8053         }
8054
8055         for (i = 0; i < n_instructions; i++) {
8056                 char *string = strdup(instructions[i]);
8057                 if (!string) {
8058                         err = ENOMEM;
8059                         goto error;
8060                 }
8061
8062                 err = instr_translate(p, a, string, &instr[i], &data[i]);
8063                 if (err) {
8064                         free(string);
8065                         goto error;
8066                 }
8067
8068                 free(string);
8069         }
8070
8071         err = instr_label_check(data, n_instructions);
8072         if (err)
8073                 goto error;
8074
8075         err = instr_verify(p, a, instr, data, n_instructions);
8076         if (err)
8077                 goto error;
8078
8079         n_instructions = instr_optimize(instr, data, n_instructions);
8080
8081         err = instr_jmp_resolve(instr, data, n_instructions);
8082         if (err)
8083                 goto error;
8084
8085         if (a) {
8086                 a->instructions = instr;
8087                 a->n_instructions = n_instructions;
8088         } else {
8089                 p->instructions = instr;
8090                 p->n_instructions = n_instructions;
8091         }
8092
8093         free(data);
8094         return 0;
8095
8096 error:
8097         free(data);
8098         free(instr);
8099         return err;
8100 }
8101
8102 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
8103
8104 static instr_exec_t instruction_table[] = {
8105         [INSTR_RX] = instr_rx_exec,
8106         [INSTR_TX] = instr_tx_exec,
8107         [INSTR_TX_I] = instr_tx_i_exec,
8108
8109         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
8110         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
8111         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
8112         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
8113         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
8114         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
8115         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
8116         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
8117
8118         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
8119         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
8120         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
8121         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
8122         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
8123         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
8124         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
8125         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
8126         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
8127
8128         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
8129         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
8130
8131         [INSTR_MOV] = instr_mov_exec,
8132         [INSTR_MOV_S] = instr_mov_s_exec,
8133         [INSTR_MOV_I] = instr_mov_i_exec,
8134
8135         [INSTR_DMA_HT] = instr_dma_ht_exec,
8136         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
8137         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
8138         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
8139         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
8140         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
8141         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
8142         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
8143
8144         [INSTR_ALU_ADD] = instr_alu_add_exec,
8145         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
8146         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
8147         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
8148         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
8149         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
8150
8151         [INSTR_ALU_SUB] = instr_alu_sub_exec,
8152         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
8153         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
8154         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
8155         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
8156         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
8157
8158         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
8159         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
8160         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
8161         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
8162
8163         [INSTR_ALU_AND] = instr_alu_and_exec,
8164         [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
8165         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
8166
8167         [INSTR_ALU_OR] = instr_alu_or_exec,
8168         [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
8169         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
8170
8171         [INSTR_ALU_XOR] = instr_alu_xor_exec,
8172         [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec,
8173         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
8174
8175         [INSTR_ALU_SHL] = instr_alu_shl_exec,
8176         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
8177         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
8178         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
8179         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
8180         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
8181
8182         [INSTR_ALU_SHR] = instr_alu_shr_exec,
8183         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
8184         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
8185         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
8186         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
8187         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
8188
8189         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
8190         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
8191         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
8192
8193         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
8194         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
8195         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
8196         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
8197         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
8198         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
8199
8200         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
8201         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
8202         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
8203         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
8204         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
8205         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
8206         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
8207         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
8208         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
8209
8210         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
8211         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
8212         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
8213         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
8214         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
8215         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
8216         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
8217         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
8218         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
8219
8220         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
8221         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
8222         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
8223
8224         [INSTR_METER_HHM] = instr_meter_hhm_exec,
8225         [INSTR_METER_HHI] = instr_meter_hhi_exec,
8226         [INSTR_METER_HMM] = instr_meter_hmm_exec,
8227         [INSTR_METER_HMI] = instr_meter_hmi_exec,
8228         [INSTR_METER_MHM] = instr_meter_mhm_exec,
8229         [INSTR_METER_MHI] = instr_meter_mhi_exec,
8230         [INSTR_METER_MMM] = instr_meter_mmm_exec,
8231         [INSTR_METER_MMI] = instr_meter_mmi_exec,
8232         [INSTR_METER_IHM] = instr_meter_ihm_exec,
8233         [INSTR_METER_IHI] = instr_meter_ihi_exec,
8234         [INSTR_METER_IMM] = instr_meter_imm_exec,
8235         [INSTR_METER_IMI] = instr_meter_imi_exec,
8236
8237         [INSTR_TABLE] = instr_table_exec,
8238         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
8239         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
8240
8241         [INSTR_JMP] = instr_jmp_exec,
8242         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
8243         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
8244         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
8245         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
8246         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
8247         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
8248
8249         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
8250         [INSTR_JMP_EQ_S] = instr_jmp_eq_s_exec,
8251         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
8252
8253         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
8254         [INSTR_JMP_NEQ_S] = instr_jmp_neq_s_exec,
8255         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
8256
8257         [INSTR_JMP_LT] = instr_jmp_lt_exec,
8258         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
8259         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
8260         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
8261         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
8262         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
8263
8264         [INSTR_JMP_GT] = instr_jmp_gt_exec,
8265         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
8266         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
8267         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
8268         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
8269         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
8270
8271         [INSTR_RETURN] = instr_return_exec,
8272 };
8273
8274 static inline void
8275 instr_exec(struct rte_swx_pipeline *p)
8276 {
8277         struct thread *t = &p->threads[p->thread_id];
8278         struct instruction *ip = t->ip;
8279         instr_exec_t instr = instruction_table[ip->type];
8280
8281         instr(p);
8282 }
8283
8284 /*
8285  * Action.
8286  */
8287 static struct action *
8288 action_find(struct rte_swx_pipeline *p, const char *name)
8289 {
8290         struct action *elem;
8291
8292         if (!name)
8293                 return NULL;
8294
8295         TAILQ_FOREACH(elem, &p->actions, node)
8296                 if (strcmp(elem->name, name) == 0)
8297                         return elem;
8298
8299         return NULL;
8300 }
8301
8302 static struct action *
8303 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8304 {
8305         struct action *action = NULL;
8306
8307         TAILQ_FOREACH(action, &p->actions, node)
8308                 if (action->id == id)
8309                         return action;
8310
8311         return NULL;
8312 }
8313
8314 static struct field *
8315 action_field_find(struct action *a, const char *name)
8316 {
8317         return a->st ? struct_type_field_find(a->st, name) : NULL;
8318 }
8319
8320 static struct field *
8321 action_field_parse(struct action *action, const char *name)
8322 {
8323         if (name[0] != 't' || name[1] != '.')
8324                 return NULL;
8325
8326         return action_field_find(action, &name[2]);
8327 }
8328
8329 int
8330 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
8331                                const char *name,
8332                                const char *args_struct_type_name,
8333                                const char **instructions,
8334                                uint32_t n_instructions)
8335 {
8336         struct struct_type *args_struct_type;
8337         struct action *a;
8338         int err;
8339
8340         CHECK(p, EINVAL);
8341
8342         CHECK_NAME(name, EINVAL);
8343         CHECK(!action_find(p, name), EEXIST);
8344
8345         if (args_struct_type_name) {
8346                 CHECK_NAME(args_struct_type_name, EINVAL);
8347                 args_struct_type = struct_type_find(p, args_struct_type_name);
8348                 CHECK(args_struct_type, EINVAL);
8349         } else {
8350                 args_struct_type = NULL;
8351         }
8352
8353         /* Node allocation. */
8354         a = calloc(1, sizeof(struct action));
8355         CHECK(a, ENOMEM);
8356
8357         /* Node initialization. */
8358         strcpy(a->name, name);
8359         a->st = args_struct_type;
8360         a->id = p->n_actions;
8361
8362         /* Instruction translation. */
8363         err = instruction_config(p, a, instructions, n_instructions);
8364         if (err) {
8365                 free(a);
8366                 return err;
8367         }
8368
8369         /* Node add to tailq. */
8370         TAILQ_INSERT_TAIL(&p->actions, a, node);
8371         p->n_actions++;
8372
8373         return 0;
8374 }
8375
8376 static int
8377 action_build(struct rte_swx_pipeline *p)
8378 {
8379         struct action *action;
8380
8381         p->action_instructions = calloc(p->n_actions,
8382                                         sizeof(struct instruction *));
8383         CHECK(p->action_instructions, ENOMEM);
8384
8385         TAILQ_FOREACH(action, &p->actions, node)
8386                 p->action_instructions[action->id] = action->instructions;
8387
8388         return 0;
8389 }
8390
8391 static void
8392 action_build_free(struct rte_swx_pipeline *p)
8393 {
8394         free(p->action_instructions);
8395         p->action_instructions = NULL;
8396 }
8397
8398 static void
8399 action_free(struct rte_swx_pipeline *p)
8400 {
8401         action_build_free(p);
8402
8403         for ( ; ; ) {
8404                 struct action *action;
8405
8406                 action = TAILQ_FIRST(&p->actions);
8407                 if (!action)
8408                         break;
8409
8410                 TAILQ_REMOVE(&p->actions, action, node);
8411                 free(action->instructions);
8412                 free(action);
8413         }
8414 }
8415
8416 /*
8417  * Table.
8418  */
8419 static struct table_type *
8420 table_type_find(struct rte_swx_pipeline *p, const char *name)
8421 {
8422         struct table_type *elem;
8423
8424         TAILQ_FOREACH(elem, &p->table_types, node)
8425                 if (strcmp(elem->name, name) == 0)
8426                         return elem;
8427
8428         return NULL;
8429 }
8430
8431 static struct table_type *
8432 table_type_resolve(struct rte_swx_pipeline *p,
8433                    const char *recommended_type_name,
8434                    enum rte_swx_table_match_type match_type)
8435 {
8436         struct table_type *elem;
8437
8438         /* Only consider the recommended type if the match type is correct. */
8439         if (recommended_type_name)
8440                 TAILQ_FOREACH(elem, &p->table_types, node)
8441                         if (!strcmp(elem->name, recommended_type_name) &&
8442                             (elem->match_type == match_type))
8443                                 return elem;
8444
8445         /* Ignore the recommended type and get the first element with this match
8446          * type.
8447          */
8448         TAILQ_FOREACH(elem, &p->table_types, node)
8449                 if (elem->match_type == match_type)
8450                         return elem;
8451
8452         return NULL;
8453 }
8454
8455 static struct table *
8456 table_find(struct rte_swx_pipeline *p, const char *name)
8457 {
8458         struct table *elem;
8459
8460         TAILQ_FOREACH(elem, &p->tables, node)
8461                 if (strcmp(elem->name, name) == 0)
8462                         return elem;
8463
8464         return NULL;
8465 }
8466
8467 static struct table *
8468 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8469 {
8470         struct table *table = NULL;
8471
8472         TAILQ_FOREACH(table, &p->tables, node)
8473                 if (table->id == id)
8474                         return table;
8475
8476         return NULL;
8477 }
8478
8479 int
8480 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
8481                                      const char *name,
8482                                      enum rte_swx_table_match_type match_type,
8483                                      struct rte_swx_table_ops *ops)
8484 {
8485         struct table_type *elem;
8486
8487         CHECK(p, EINVAL);
8488
8489         CHECK_NAME(name, EINVAL);
8490         CHECK(!table_type_find(p, name), EEXIST);
8491
8492         CHECK(ops, EINVAL);
8493         CHECK(ops->create, EINVAL);
8494         CHECK(ops->lkp, EINVAL);
8495         CHECK(ops->free, EINVAL);
8496
8497         /* Node allocation. */
8498         elem = calloc(1, sizeof(struct table_type));
8499         CHECK(elem, ENOMEM);
8500
8501         /* Node initialization. */
8502         strcpy(elem->name, name);
8503         elem->match_type = match_type;
8504         memcpy(&elem->ops, ops, sizeof(*ops));
8505
8506         /* Node add to tailq. */
8507         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
8508
8509         return 0;
8510 }
8511
8512 static enum rte_swx_table_match_type
8513 table_match_type_resolve(struct rte_swx_match_field_params *fields,
8514                          uint32_t n_fields,
8515                          uint32_t max_offset_field_id)
8516 {
8517         uint32_t n_fields_em = 0, i;
8518
8519         for (i = 0; i < n_fields; i++)
8520                 if (fields[i].match_type == RTE_SWX_TABLE_MATCH_EXACT)
8521                         n_fields_em++;
8522
8523         if (n_fields_em == n_fields)
8524                 return RTE_SWX_TABLE_MATCH_EXACT;
8525
8526         if ((n_fields_em == n_fields - 1) &&
8527             (fields[max_offset_field_id].match_type == RTE_SWX_TABLE_MATCH_LPM))
8528                 return RTE_SWX_TABLE_MATCH_LPM;
8529
8530         return RTE_SWX_TABLE_MATCH_WILDCARD;
8531 }
8532
8533 static int
8534 table_match_fields_check(struct rte_swx_pipeline *p,
8535                          struct rte_swx_pipeline_table_params *params,
8536                          struct header **header,
8537                          uint32_t *min_offset_field_id,
8538                          uint32_t *max_offset_field_id)
8539 {
8540         struct header *h0 = NULL;
8541         struct field *hf, *mf;
8542         uint32_t *offset = NULL, min_offset, max_offset, min_offset_pos, max_offset_pos, i;
8543         int status = 0;
8544
8545         /* Return if no match fields. */
8546         if (!params->n_fields) {
8547                 if (params->fields) {
8548                         status = -EINVAL;
8549                         goto end;
8550                 }
8551
8552                 return 0;
8553         }
8554
8555         /* Memory allocation. */
8556         offset = calloc(params->n_fields, sizeof(uint32_t));
8557         if (!offset) {
8558                 status = -ENOMEM;
8559                 goto end;
8560         }
8561
8562         /* Check that all the match fields belong to either the same header or
8563          * to the meta-data.
8564          */
8565         hf = header_field_parse(p, params->fields[0].name, &h0);
8566         mf = metadata_field_parse(p, params->fields[0].name);
8567         if (!hf && !mf) {
8568                 status = -EINVAL;
8569                 goto end;
8570         }
8571
8572         offset[0] = h0 ? hf->offset : mf->offset;
8573
8574         for (i = 1; i < params->n_fields; i++)
8575                 if (h0) {
8576                         struct header *h;
8577
8578                         hf = header_field_parse(p, params->fields[i].name, &h);
8579                         if (!hf || (h->id != h0->id)) {
8580                                 status = -EINVAL;
8581                                 goto end;
8582                         }
8583
8584                         offset[i] = hf->offset;
8585                 } else {
8586                         mf = metadata_field_parse(p, params->fields[i].name);
8587                         if (!mf) {
8588                                 status = -EINVAL;
8589                                 goto end;
8590                         }
8591
8592                         offset[i] = mf->offset;
8593                 }
8594
8595         /* Check that there are no duplicated match fields. */
8596         for (i = 0; i < params->n_fields; i++) {
8597                 uint32_t j;
8598
8599                 for (j = 0; j < i; j++)
8600                         if (offset[j] == offset[i]) {
8601                                 status = -EINVAL;
8602                                 goto end;
8603                         }
8604         }
8605
8606         /* Find the min and max offset fields. */
8607         min_offset = offset[0];
8608         max_offset = offset[0];
8609         min_offset_pos = 0;
8610         max_offset_pos = 0;
8611
8612         for (i = 1; i < params->n_fields; i++) {
8613                 if (offset[i] < min_offset) {
8614                         min_offset = offset[i];
8615                         min_offset_pos = i;
8616                 }
8617
8618                 if (offset[i] > max_offset) {
8619                         max_offset = offset[i];
8620                         max_offset_pos = i;
8621                 }
8622         }
8623
8624         /* Return. */
8625         if (header)
8626                 *header = h0;
8627
8628         if (min_offset_field_id)
8629                 *min_offset_field_id = min_offset_pos;
8630
8631         if (max_offset_field_id)
8632                 *max_offset_field_id = max_offset_pos;
8633
8634 end:
8635         free(offset);
8636         return status;
8637 }
8638
8639 int
8640 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8641                               const char *name,
8642                               struct rte_swx_pipeline_table_params *params,
8643                               const char *recommended_table_type_name,
8644                               const char *args,
8645                               uint32_t size)
8646 {
8647         struct table_type *type;
8648         struct table *t;
8649         struct action *default_action;
8650         struct header *header = NULL;
8651         uint32_t action_data_size_max = 0, min_offset_field_id = 0, max_offset_field_id = 0, i;
8652         int status = 0;
8653
8654         CHECK(p, EINVAL);
8655
8656         CHECK_NAME(name, EINVAL);
8657         CHECK(!table_find(p, name), EEXIST);
8658
8659         CHECK(params, EINVAL);
8660
8661         /* Match checks. */
8662         status = table_match_fields_check(p,
8663                                           params,
8664                                           &header,
8665                                           &min_offset_field_id,
8666                                           &max_offset_field_id);
8667         if (status)
8668                 return status;
8669
8670         /* Action checks. */
8671         CHECK(params->n_actions, EINVAL);
8672         CHECK(params->action_names, EINVAL);
8673         for (i = 0; i < params->n_actions; i++) {
8674                 const char *action_name = params->action_names[i];
8675                 struct action *a;
8676                 uint32_t action_data_size;
8677
8678                 CHECK_NAME(action_name, EINVAL);
8679
8680                 a = action_find(p, action_name);
8681                 CHECK(a, EINVAL);
8682
8683                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8684                 if (action_data_size > action_data_size_max)
8685                         action_data_size_max = action_data_size;
8686         }
8687
8688         CHECK_NAME(params->default_action_name, EINVAL);
8689         for (i = 0; i < p->n_actions; i++)
8690                 if (!strcmp(params->action_names[i],
8691                             params->default_action_name))
8692                         break;
8693         CHECK(i < params->n_actions, EINVAL);
8694         default_action = action_find(p, params->default_action_name);
8695         CHECK((default_action->st && params->default_action_data) ||
8696               !params->default_action_data, EINVAL);
8697
8698         /* Table type checks. */
8699         if (recommended_table_type_name)
8700                 CHECK_NAME(recommended_table_type_name, EINVAL);
8701
8702         if (params->n_fields) {
8703                 enum rte_swx_table_match_type match_type;
8704
8705                 match_type = table_match_type_resolve(params->fields,
8706                                                       params->n_fields,
8707                                                       max_offset_field_id);
8708                 type = table_type_resolve(p,
8709                                           recommended_table_type_name,
8710                                           match_type);
8711                 CHECK(type, EINVAL);
8712         } else {
8713                 type = NULL;
8714         }
8715
8716         /* Memory allocation. */
8717         t = calloc(1, sizeof(struct table));
8718         CHECK(t, ENOMEM);
8719
8720         t->fields = calloc(params->n_fields, sizeof(struct match_field));
8721         if (!t->fields) {
8722                 free(t);
8723                 CHECK(0, ENOMEM);
8724         }
8725
8726         t->actions = calloc(params->n_actions, sizeof(struct action *));
8727         if (!t->actions) {
8728                 free(t->fields);
8729                 free(t);
8730                 CHECK(0, ENOMEM);
8731         }
8732
8733         if (action_data_size_max) {
8734                 t->default_action_data = calloc(1, action_data_size_max);
8735                 if (!t->default_action_data) {
8736                         free(t->actions);
8737                         free(t->fields);
8738                         free(t);
8739                         CHECK(0, ENOMEM);
8740                 }
8741         }
8742
8743         /* Node initialization. */
8744         strcpy(t->name, name);
8745         if (args && args[0])
8746                 strcpy(t->args, args);
8747         t->type = type;
8748
8749         for (i = 0; i < params->n_fields; i++) {
8750                 struct rte_swx_match_field_params *field = &params->fields[i];
8751                 struct match_field *f = &t->fields[i];
8752
8753                 f->match_type = field->match_type;
8754                 f->field = header ?
8755                         header_field_parse(p, field->name, NULL) :
8756                         metadata_field_parse(p, field->name);
8757         }
8758         t->n_fields = params->n_fields;
8759         t->header = header;
8760
8761         for (i = 0; i < params->n_actions; i++)
8762                 t->actions[i] = action_find(p, params->action_names[i]);
8763         t->default_action = default_action;
8764         if (default_action->st)
8765                 memcpy(t->default_action_data,
8766                        params->default_action_data,
8767                        default_action->st->n_bits / 8);
8768         t->n_actions = params->n_actions;
8769         t->default_action_is_const = params->default_action_is_const;
8770         t->action_data_size_max = action_data_size_max;
8771
8772         t->size = size;
8773         t->id = p->n_tables;
8774
8775         /* Node add to tailq. */
8776         TAILQ_INSERT_TAIL(&p->tables, t, node);
8777         p->n_tables++;
8778
8779         return 0;
8780 }
8781
8782 static struct rte_swx_table_params *
8783 table_params_get(struct table *table)
8784 {
8785         struct rte_swx_table_params *params;
8786         struct field *first, *last;
8787         uint8_t *key_mask;
8788         uint32_t key_size, key_offset, action_data_size, i;
8789
8790         /* Memory allocation. */
8791         params = calloc(1, sizeof(struct rte_swx_table_params));
8792         if (!params)
8793                 return NULL;
8794
8795         /* Find first (smallest offset) and last (biggest offset) match fields. */
8796         first = table->fields[0].field;
8797         last = table->fields[0].field;
8798
8799         for (i = 0; i < table->n_fields; i++) {
8800                 struct field *f = table->fields[i].field;
8801
8802                 if (f->offset < first->offset)
8803                         first = f;
8804
8805                 if (f->offset > last->offset)
8806                         last = f;
8807         }
8808
8809         /* Key offset and size. */
8810         key_offset = first->offset / 8;
8811         key_size = (last->offset + last->n_bits - first->offset) / 8;
8812
8813         /* Memory allocation. */
8814         key_mask = calloc(1, key_size);
8815         if (!key_mask) {
8816                 free(params);
8817                 return NULL;
8818         }
8819
8820         /* Key mask. */
8821         for (i = 0; i < table->n_fields; i++) {
8822                 struct field *f = table->fields[i].field;
8823                 uint32_t start = (f->offset - first->offset) / 8;
8824                 size_t size = f->n_bits / 8;
8825
8826                 memset(&key_mask[start], 0xFF, size);
8827         }
8828
8829         /* Action data size. */
8830         action_data_size = 0;
8831         for (i = 0; i < table->n_actions; i++) {
8832                 struct action *action = table->actions[i];
8833                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8834
8835                 if (ads > action_data_size)
8836                         action_data_size = ads;
8837         }
8838
8839         /* Fill in. */
8840         params->match_type = table->type->match_type;
8841         params->key_size = key_size;
8842         params->key_offset = key_offset;
8843         params->key_mask0 = key_mask;
8844         params->action_data_size = action_data_size;
8845         params->n_keys_max = table->size;
8846
8847         return params;
8848 }
8849
8850 static void
8851 table_params_free(struct rte_swx_table_params *params)
8852 {
8853         if (!params)
8854                 return;
8855
8856         free(params->key_mask0);
8857         free(params);
8858 }
8859
8860 static int
8861 table_state_build(struct rte_swx_pipeline *p)
8862 {
8863         struct table *table;
8864
8865         p->table_state = calloc(p->n_tables,
8866                                 sizeof(struct rte_swx_table_state));
8867         CHECK(p->table_state, ENOMEM);
8868
8869         TAILQ_FOREACH(table, &p->tables, node) {
8870                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8871
8872                 if (table->type) {
8873                         struct rte_swx_table_params *params;
8874
8875                         /* ts->obj. */
8876                         params = table_params_get(table);
8877                         CHECK(params, ENOMEM);
8878
8879                         ts->obj = table->type->ops.create(params,
8880                                 NULL,
8881                                 table->args,
8882                                 p->numa_node);
8883
8884                         table_params_free(params);
8885                         CHECK(ts->obj, ENODEV);
8886                 }
8887
8888                 /* ts->default_action_data. */
8889                 if (table->action_data_size_max) {
8890                         ts->default_action_data =
8891                                 malloc(table->action_data_size_max);
8892                         CHECK(ts->default_action_data, ENOMEM);
8893
8894                         memcpy(ts->default_action_data,
8895                                table->default_action_data,
8896                                table->action_data_size_max);
8897                 }
8898
8899                 /* ts->default_action_id. */
8900                 ts->default_action_id = table->default_action->id;
8901         }
8902
8903         return 0;
8904 }
8905
8906 static void
8907 table_state_build_free(struct rte_swx_pipeline *p)
8908 {
8909         uint32_t i;
8910
8911         if (!p->table_state)
8912                 return;
8913
8914         for (i = 0; i < p->n_tables; i++) {
8915                 struct rte_swx_table_state *ts = &p->table_state[i];
8916                 struct table *table = table_find_by_id(p, i);
8917
8918                 /* ts->obj. */
8919                 if (table->type && ts->obj)
8920                         table->type->ops.free(ts->obj);
8921
8922                 /* ts->default_action_data. */
8923                 free(ts->default_action_data);
8924         }
8925
8926         free(p->table_state);
8927         p->table_state = NULL;
8928 }
8929
8930 static void
8931 table_state_free(struct rte_swx_pipeline *p)
8932 {
8933         table_state_build_free(p);
8934 }
8935
8936 static int
8937 table_stub_lkp(void *table __rte_unused,
8938                void *mailbox __rte_unused,
8939                uint8_t **key __rte_unused,
8940                uint64_t *action_id __rte_unused,
8941                uint8_t **action_data __rte_unused,
8942                int *hit)
8943 {
8944         *hit = 0;
8945         return 1; /* DONE. */
8946 }
8947
8948 static int
8949 table_build(struct rte_swx_pipeline *p)
8950 {
8951         uint32_t i;
8952
8953         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8954                 struct thread *t = &p->threads[i];
8955                 struct table *table;
8956
8957                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8958                 CHECK(t->tables, ENOMEM);
8959
8960                 TAILQ_FOREACH(table, &p->tables, node) {
8961                         struct table_runtime *r = &t->tables[table->id];
8962
8963                         if (table->type) {
8964                                 uint64_t size;
8965
8966                                 size = table->type->ops.mailbox_size_get();
8967
8968                                 /* r->func. */
8969                                 r->func = table->type->ops.lkp;
8970
8971                                 /* r->mailbox. */
8972                                 if (size) {
8973                                         r->mailbox = calloc(1, size);
8974                                         CHECK(r->mailbox, ENOMEM);
8975                                 }
8976
8977                                 /* r->key. */
8978                                 r->key = table->header ?
8979                                         &t->structs[table->header->struct_id] :
8980                                         &t->structs[p->metadata_struct_id];
8981                         } else {
8982                                 r->func = table_stub_lkp;
8983                         }
8984                 }
8985         }
8986
8987         return 0;
8988 }
8989
8990 static void
8991 table_build_free(struct rte_swx_pipeline *p)
8992 {
8993         uint32_t i;
8994
8995         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8996                 struct thread *t = &p->threads[i];
8997                 uint32_t j;
8998
8999                 if (!t->tables)
9000                         continue;
9001
9002                 for (j = 0; j < p->n_tables; j++) {
9003                         struct table_runtime *r = &t->tables[j];
9004
9005                         free(r->mailbox);
9006                 }
9007
9008                 free(t->tables);
9009                 t->tables = NULL;
9010         }
9011 }
9012
9013 static void
9014 table_free(struct rte_swx_pipeline *p)
9015 {
9016         table_build_free(p);
9017
9018         /* Tables. */
9019         for ( ; ; ) {
9020                 struct table *elem;
9021
9022                 elem = TAILQ_FIRST(&p->tables);
9023                 if (!elem)
9024                         break;
9025
9026                 TAILQ_REMOVE(&p->tables, elem, node);
9027                 free(elem->fields);
9028                 free(elem->actions);
9029                 free(elem->default_action_data);
9030                 free(elem);
9031         }
9032
9033         /* Table types. */
9034         for ( ; ; ) {
9035                 struct table_type *elem;
9036
9037                 elem = TAILQ_FIRST(&p->table_types);
9038                 if (!elem)
9039                         break;
9040
9041                 TAILQ_REMOVE(&p->table_types, elem, node);
9042                 free(elem);
9043         }
9044 }
9045
9046 /*
9047  * Register array.
9048  */
9049 static struct regarray *
9050 regarray_find(struct rte_swx_pipeline *p, const char *name)
9051 {
9052         struct regarray *elem;
9053
9054         TAILQ_FOREACH(elem, &p->regarrays, node)
9055                 if (!strcmp(elem->name, name))
9056                         return elem;
9057
9058         return NULL;
9059 }
9060
9061 static struct regarray *
9062 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9063 {
9064         struct regarray *elem = NULL;
9065
9066         TAILQ_FOREACH(elem, &p->regarrays, node)
9067                 if (elem->id == id)
9068                         return elem;
9069
9070         return NULL;
9071 }
9072
9073 int
9074 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9075                               const char *name,
9076                               uint32_t size,
9077                               uint64_t init_val)
9078 {
9079         struct regarray *r;
9080
9081         CHECK(p, EINVAL);
9082
9083         CHECK_NAME(name, EINVAL);
9084         CHECK(!regarray_find(p, name), EEXIST);
9085
9086         CHECK(size, EINVAL);
9087         size = rte_align32pow2(size);
9088
9089         /* Memory allocation. */
9090         r = calloc(1, sizeof(struct regarray));
9091         CHECK(r, ENOMEM);
9092
9093         /* Node initialization. */
9094         strcpy(r->name, name);
9095         r->init_val = init_val;
9096         r->size = size;
9097         r->id = p->n_regarrays;
9098
9099         /* Node add to tailq. */
9100         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9101         p->n_regarrays++;
9102
9103         return 0;
9104 }
9105
9106 static int
9107 regarray_build(struct rte_swx_pipeline *p)
9108 {
9109         struct regarray *regarray;
9110
9111         if (!p->n_regarrays)
9112                 return 0;
9113
9114         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9115         CHECK(p->regarray_runtime, ENOMEM);
9116
9117         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9118                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9119                 uint32_t i;
9120
9121                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9122                                          RTE_CACHE_LINE_SIZE,
9123                                          p->numa_node);
9124                 CHECK(r->regarray, ENOMEM);
9125
9126                 if (regarray->init_val)
9127                         for (i = 0; i < regarray->size; i++)
9128                                 r->regarray[i] = regarray->init_val;
9129
9130                 r->size_mask = regarray->size - 1;
9131         }
9132
9133         return 0;
9134 }
9135
9136 static void
9137 regarray_build_free(struct rte_swx_pipeline *p)
9138 {
9139         uint32_t i;
9140
9141         if (!p->regarray_runtime)
9142                 return;
9143
9144         for (i = 0; i < p->n_regarrays; i++) {
9145                 struct regarray *regarray = regarray_find_by_id(p, i);
9146                 struct regarray_runtime *r = &p->regarray_runtime[i];
9147
9148                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9149         }
9150
9151         free(p->regarray_runtime);
9152         p->regarray_runtime = NULL;
9153 }
9154
9155 static void
9156 regarray_free(struct rte_swx_pipeline *p)
9157 {
9158         regarray_build_free(p);
9159
9160         for ( ; ; ) {
9161                 struct regarray *elem;
9162
9163                 elem = TAILQ_FIRST(&p->regarrays);
9164                 if (!elem)
9165                         break;
9166
9167                 TAILQ_REMOVE(&p->regarrays, elem, node);
9168                 free(elem);
9169         }
9170 }
9171
9172 /*
9173  * Meter array.
9174  */
9175 static struct meter_profile *
9176 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9177 {
9178         struct meter_profile *elem;
9179
9180         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9181                 if (!strcmp(elem->name, name))
9182                         return elem;
9183
9184         return NULL;
9185 }
9186
9187 static struct metarray *
9188 metarray_find(struct rte_swx_pipeline *p, const char *name)
9189 {
9190         struct metarray *elem;
9191
9192         TAILQ_FOREACH(elem, &p->metarrays, node)
9193                 if (!strcmp(elem->name, name))
9194                         return elem;
9195
9196         return NULL;
9197 }
9198
9199 static struct metarray *
9200 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9201 {
9202         struct metarray *elem = NULL;
9203
9204         TAILQ_FOREACH(elem, &p->metarrays, node)
9205                 if (elem->id == id)
9206                         return elem;
9207
9208         return NULL;
9209 }
9210
9211 int
9212 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9213                                  const char *name,
9214                                  uint32_t size)
9215 {
9216         struct metarray *m;
9217
9218         CHECK(p, EINVAL);
9219
9220         CHECK_NAME(name, EINVAL);
9221         CHECK(!metarray_find(p, name), EEXIST);
9222
9223         CHECK(size, EINVAL);
9224         size = rte_align32pow2(size);
9225
9226         /* Memory allocation. */
9227         m = calloc(1, sizeof(struct metarray));
9228         CHECK(m, ENOMEM);
9229
9230         /* Node initialization. */
9231         strcpy(m->name, name);
9232         m->size = size;
9233         m->id = p->n_metarrays;
9234
9235         /* Node add to tailq. */
9236         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9237         p->n_metarrays++;
9238
9239         return 0;
9240 }
9241
9242 struct meter_profile meter_profile_default = {
9243         .node = {0},
9244         .name = "",
9245         .params = {0},
9246
9247         .profile = {
9248                 .cbs = 10000,
9249                 .pbs = 10000,
9250                 .cir_period = 1,
9251                 .cir_bytes_per_period = 1,
9252                 .pir_period = 1,
9253                 .pir_bytes_per_period = 1,
9254         },
9255
9256         .n_users = 0,
9257 };
9258
9259 static void
9260 meter_init(struct meter *m)
9261 {
9262         memset(m, 0, sizeof(struct meter));
9263         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9264         m->profile = &meter_profile_default;
9265         m->color_mask = RTE_COLOR_GREEN;
9266
9267         meter_profile_default.n_users++;
9268 }
9269
9270 static int
9271 metarray_build(struct rte_swx_pipeline *p)
9272 {
9273         struct metarray *m;
9274
9275         if (!p->n_metarrays)
9276                 return 0;
9277
9278         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9279         CHECK(p->metarray_runtime, ENOMEM);
9280
9281         TAILQ_FOREACH(m, &p->metarrays, node) {
9282                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9283                 uint32_t i;
9284
9285                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9286                                          RTE_CACHE_LINE_SIZE,
9287                                          p->numa_node);
9288                 CHECK(r->metarray, ENOMEM);
9289
9290                 for (i = 0; i < m->size; i++)
9291                         meter_init(&r->metarray[i]);
9292
9293                 r->size_mask = m->size - 1;
9294         }
9295
9296         return 0;
9297 }
9298
9299 static void
9300 metarray_build_free(struct rte_swx_pipeline *p)
9301 {
9302         uint32_t i;
9303
9304         if (!p->metarray_runtime)
9305                 return;
9306
9307         for (i = 0; i < p->n_metarrays; i++) {
9308                 struct metarray *m = metarray_find_by_id(p, i);
9309                 struct metarray_runtime *r = &p->metarray_runtime[i];
9310
9311                 env_free(r->metarray, m->size * sizeof(struct meter));
9312         }
9313
9314         free(p->metarray_runtime);
9315         p->metarray_runtime = NULL;
9316 }
9317
9318 static void
9319 metarray_free(struct rte_swx_pipeline *p)
9320 {
9321         metarray_build_free(p);
9322
9323         /* Meter arrays. */
9324         for ( ; ; ) {
9325                 struct metarray *elem;
9326
9327                 elem = TAILQ_FIRST(&p->metarrays);
9328                 if (!elem)
9329                         break;
9330
9331                 TAILQ_REMOVE(&p->metarrays, elem, node);
9332                 free(elem);
9333         }
9334
9335         /* Meter profiles. */
9336         for ( ; ; ) {
9337                 struct meter_profile *elem;
9338
9339                 elem = TAILQ_FIRST(&p->meter_profiles);
9340                 if (!elem)
9341                         break;
9342
9343                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9344                 free(elem);
9345         }
9346 }
9347
9348 /*
9349  * Pipeline.
9350  */
9351 int
9352 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9353 {
9354         struct rte_swx_pipeline *pipeline;
9355
9356         /* Check input parameters. */
9357         CHECK(p, EINVAL);
9358
9359         /* Memory allocation. */
9360         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9361         CHECK(pipeline, ENOMEM);
9362
9363         /* Initialization. */
9364         TAILQ_INIT(&pipeline->struct_types);
9365         TAILQ_INIT(&pipeline->port_in_types);
9366         TAILQ_INIT(&pipeline->ports_in);
9367         TAILQ_INIT(&pipeline->port_out_types);
9368         TAILQ_INIT(&pipeline->ports_out);
9369         TAILQ_INIT(&pipeline->extern_types);
9370         TAILQ_INIT(&pipeline->extern_objs);
9371         TAILQ_INIT(&pipeline->extern_funcs);
9372         TAILQ_INIT(&pipeline->headers);
9373         TAILQ_INIT(&pipeline->actions);
9374         TAILQ_INIT(&pipeline->table_types);
9375         TAILQ_INIT(&pipeline->tables);
9376         TAILQ_INIT(&pipeline->regarrays);
9377         TAILQ_INIT(&pipeline->meter_profiles);
9378         TAILQ_INIT(&pipeline->metarrays);
9379
9380         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9381         pipeline->numa_node = numa_node;
9382
9383         *p = pipeline;
9384         return 0;
9385 }
9386
9387 void
9388 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9389 {
9390         if (!p)
9391                 return;
9392
9393         free(p->instructions);
9394
9395         metarray_free(p);
9396         regarray_free(p);
9397         table_state_free(p);
9398         table_free(p);
9399         action_free(p);
9400         metadata_free(p);
9401         header_free(p);
9402         extern_func_free(p);
9403         extern_obj_free(p);
9404         port_out_free(p);
9405         port_in_free(p);
9406         struct_free(p);
9407
9408         free(p);
9409 }
9410
9411 int
9412 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9413                                      const char **instructions,
9414                                      uint32_t n_instructions)
9415 {
9416         int err;
9417         uint32_t i;
9418
9419         err = instruction_config(p, NULL, instructions, n_instructions);
9420         if (err)
9421                 return err;
9422
9423         /* Thread instruction pointer reset. */
9424         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9425                 struct thread *t = &p->threads[i];
9426
9427                 thread_ip_reset(p, t);
9428         }
9429
9430         return 0;
9431 }
9432
9433 int
9434 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9435 {
9436         int status;
9437
9438         CHECK(p, EINVAL);
9439         CHECK(p->build_done == 0, EEXIST);
9440
9441         status = port_in_build(p);
9442         if (status)
9443                 goto error;
9444
9445         status = port_out_build(p);
9446         if (status)
9447                 goto error;
9448
9449         status = struct_build(p);
9450         if (status)
9451                 goto error;
9452
9453         status = extern_obj_build(p);
9454         if (status)
9455                 goto error;
9456
9457         status = extern_func_build(p);
9458         if (status)
9459                 goto error;
9460
9461         status = header_build(p);
9462         if (status)
9463                 goto error;
9464
9465         status = metadata_build(p);
9466         if (status)
9467                 goto error;
9468
9469         status = action_build(p);
9470         if (status)
9471                 goto error;
9472
9473         status = table_build(p);
9474         if (status)
9475                 goto error;
9476
9477         status = table_state_build(p);
9478         if (status)
9479                 goto error;
9480
9481         status = regarray_build(p);
9482         if (status)
9483                 goto error;
9484
9485         status = metarray_build(p);
9486         if (status)
9487                 goto error;
9488
9489         p->build_done = 1;
9490         return 0;
9491
9492 error:
9493         metarray_build_free(p);
9494         regarray_build_free(p);
9495         table_state_build_free(p);
9496         table_build_free(p);
9497         action_build_free(p);
9498         metadata_build_free(p);
9499         header_build_free(p);
9500         extern_func_build_free(p);
9501         extern_obj_build_free(p);
9502         port_out_build_free(p);
9503         port_in_build_free(p);
9504         struct_build_free(p);
9505
9506         return status;
9507 }
9508
9509 void
9510 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9511 {
9512         uint32_t i;
9513
9514         for (i = 0; i < n_instructions; i++)
9515                 instr_exec(p);
9516 }
9517
9518 void
9519 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9520 {
9521         uint32_t i;
9522
9523         for (i = 0; i < p->n_ports_out; i++) {
9524                 struct port_out_runtime *port = &p->out[i];
9525
9526                 if (port->flush)
9527                         port->flush(port->obj);
9528         }
9529 }
9530
9531 /*
9532  * Control.
9533  */
9534 int
9535 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9536                               struct rte_swx_ctl_pipeline_info *pipeline)
9537 {
9538         struct action *action;
9539         struct table *table;
9540         uint32_t n_actions = 0, n_tables = 0;
9541
9542         if (!p || !pipeline)
9543                 return -EINVAL;
9544
9545         TAILQ_FOREACH(action, &p->actions, node)
9546                 n_actions++;
9547
9548         TAILQ_FOREACH(table, &p->tables, node)
9549                 n_tables++;
9550
9551         pipeline->n_ports_in = p->n_ports_in;
9552         pipeline->n_ports_out = p->n_ports_out;
9553         pipeline->n_actions = n_actions;
9554         pipeline->n_tables = n_tables;
9555         pipeline->n_regarrays = p->n_regarrays;
9556         pipeline->n_metarrays = p->n_metarrays;
9557
9558         return 0;
9559 }
9560
9561 int
9562 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9563 {
9564         if (!p || !numa_node)
9565                 return -EINVAL;
9566
9567         *numa_node = p->numa_node;
9568         return 0;
9569 }
9570
9571 int
9572 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9573                             uint32_t action_id,
9574                             struct rte_swx_ctl_action_info *action)
9575 {
9576         struct action *a = NULL;
9577
9578         if (!p || (action_id >= p->n_actions) || !action)
9579                 return -EINVAL;
9580
9581         a = action_find_by_id(p, action_id);
9582         if (!a)
9583                 return -EINVAL;
9584
9585         strcpy(action->name, a->name);
9586         action->n_args = a->st ? a->st->n_fields : 0;
9587         return 0;
9588 }
9589
9590 int
9591 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9592                                 uint32_t action_id,
9593                                 uint32_t action_arg_id,
9594                                 struct rte_swx_ctl_action_arg_info *action_arg)
9595 {
9596         struct action *a = NULL;
9597         struct field *arg = NULL;
9598
9599         if (!p || (action_id >= p->n_actions) || !action_arg)
9600                 return -EINVAL;
9601
9602         a = action_find_by_id(p, action_id);
9603         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9604                 return -EINVAL;
9605
9606         arg = &a->st->fields[action_arg_id];
9607         strcpy(action_arg->name, arg->name);
9608         action_arg->n_bits = arg->n_bits;
9609
9610         return 0;
9611 }
9612
9613 int
9614 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9615                            uint32_t table_id,
9616                            struct rte_swx_ctl_table_info *table)
9617 {
9618         struct table *t = NULL;
9619
9620         if (!p || !table)
9621                 return -EINVAL;
9622
9623         t = table_find_by_id(p, table_id);
9624         if (!t)
9625                 return -EINVAL;
9626
9627         strcpy(table->name, t->name);
9628         strcpy(table->args, t->args);
9629         table->n_match_fields = t->n_fields;
9630         table->n_actions = t->n_actions;
9631         table->default_action_is_const = t->default_action_is_const;
9632         table->size = t->size;
9633         return 0;
9634 }
9635
9636 int
9637 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9638         uint32_t table_id,
9639         uint32_t match_field_id,
9640         struct rte_swx_ctl_table_match_field_info *match_field)
9641 {
9642         struct table *t;
9643         struct match_field *f;
9644
9645         if (!p || (table_id >= p->n_tables) || !match_field)
9646                 return -EINVAL;
9647
9648         t = table_find_by_id(p, table_id);
9649         if (!t || (match_field_id >= t->n_fields))
9650                 return -EINVAL;
9651
9652         f = &t->fields[match_field_id];
9653         match_field->match_type = f->match_type;
9654         match_field->is_header = t->header ? 1 : 0;
9655         match_field->n_bits = f->field->n_bits;
9656         match_field->offset = f->field->offset;
9657
9658         return 0;
9659 }
9660
9661 int
9662 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9663         uint32_t table_id,
9664         uint32_t table_action_id,
9665         struct rte_swx_ctl_table_action_info *table_action)
9666 {
9667         struct table *t;
9668
9669         if (!p || (table_id >= p->n_tables) || !table_action)
9670                 return -EINVAL;
9671
9672         t = table_find_by_id(p, table_id);
9673         if (!t || (table_action_id >= t->n_actions))
9674                 return -EINVAL;
9675
9676         table_action->action_id = t->actions[table_action_id]->id;
9677
9678         return 0;
9679 }
9680
9681 int
9682 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9683                           uint32_t table_id,
9684                           struct rte_swx_table_ops *table_ops,
9685                           int *is_stub)
9686 {
9687         struct table *t;
9688
9689         if (!p || (table_id >= p->n_tables))
9690                 return -EINVAL;
9691
9692         t = table_find_by_id(p, table_id);
9693         if (!t)
9694                 return -EINVAL;
9695
9696         if (t->type) {
9697                 if (table_ops)
9698                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9699                 *is_stub = 0;
9700         } else {
9701                 *is_stub = 1;
9702         }
9703
9704         return 0;
9705 }
9706
9707 int
9708 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9709                                  struct rte_swx_table_state **table_state)
9710 {
9711         if (!p || !table_state || !p->build_done)
9712                 return -EINVAL;
9713
9714         *table_state = p->table_state;
9715         return 0;
9716 }
9717
9718 int
9719 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9720                                  struct rte_swx_table_state *table_state)
9721 {
9722         if (!p || !table_state || !p->build_done)
9723                 return -EINVAL;
9724
9725         p->table_state = table_state;
9726         return 0;
9727 }
9728
9729 int
9730 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9731                                         uint32_t port_id,
9732                                         struct rte_swx_port_in_stats *stats)
9733 {
9734         struct port_in *port;
9735
9736         if (!p || !stats)
9737                 return -EINVAL;
9738
9739         port = port_in_find(p, port_id);
9740         if (!port)
9741                 return -EINVAL;
9742
9743         port->type->ops.stats_read(port->obj, stats);
9744         return 0;
9745 }
9746
9747 int
9748 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9749                                          uint32_t port_id,
9750                                          struct rte_swx_port_out_stats *stats)
9751 {
9752         struct port_out *port;
9753
9754         if (!p || !stats)
9755                 return -EINVAL;
9756
9757         port = port_out_find(p, port_id);
9758         if (!port)
9759                 return -EINVAL;
9760
9761         port->type->ops.stats_read(port->obj, stats);
9762         return 0;
9763 }
9764
9765 int
9766 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9767                               uint32_t regarray_id,
9768                               struct rte_swx_ctl_regarray_info *regarray)
9769 {
9770         struct regarray *r;
9771
9772         if (!p || !regarray)
9773                 return -EINVAL;
9774
9775         r = regarray_find_by_id(p, regarray_id);
9776         if (!r)
9777                 return -EINVAL;
9778
9779         strcpy(regarray->name, r->name);
9780         regarray->size = r->size;
9781         return 0;
9782 }
9783
9784 int
9785 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9786                                    const char *regarray_name,
9787                                    uint32_t regarray_index,
9788                                    uint64_t *value)
9789 {
9790         struct regarray *regarray;
9791         struct regarray_runtime *r;
9792
9793         if (!p || !regarray_name || !value)
9794                 return -EINVAL;
9795
9796         regarray = regarray_find(p, regarray_name);
9797         if (!regarray || (regarray_index >= regarray->size))
9798                 return -EINVAL;
9799
9800         r = &p->regarray_runtime[regarray->id];
9801         *value = r->regarray[regarray_index];
9802         return 0;
9803 }
9804
9805 int
9806 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9807                                    const char *regarray_name,
9808                                    uint32_t regarray_index,
9809                                    uint64_t value)
9810 {
9811         struct regarray *regarray;
9812         struct regarray_runtime *r;
9813
9814         if (!p || !regarray_name)
9815                 return -EINVAL;
9816
9817         regarray = regarray_find(p, regarray_name);
9818         if (!regarray || (regarray_index >= regarray->size))
9819                 return -EINVAL;
9820
9821         r = &p->regarray_runtime[regarray->id];
9822         r->regarray[regarray_index] = value;
9823         return 0;
9824 }
9825
9826 int
9827 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9828                               uint32_t metarray_id,
9829                               struct rte_swx_ctl_metarray_info *metarray)
9830 {
9831         struct metarray *m;
9832
9833         if (!p || !metarray)
9834                 return -EINVAL;
9835
9836         m = metarray_find_by_id(p, metarray_id);
9837         if (!m)
9838                 return -EINVAL;
9839
9840         strcpy(metarray->name, m->name);
9841         metarray->size = m->size;
9842         return 0;
9843 }
9844
9845 int
9846 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9847                               const char *name,
9848                               struct rte_meter_trtcm_params *params)
9849 {
9850         struct meter_profile *mp;
9851         int status;
9852
9853         CHECK(p, EINVAL);
9854         CHECK_NAME(name, EINVAL);
9855         CHECK(params, EINVAL);
9856         CHECK(!meter_profile_find(p, name), EEXIST);
9857
9858         /* Node allocation. */
9859         mp = calloc(1, sizeof(struct meter_profile));
9860         CHECK(mp, ENOMEM);
9861
9862         /* Node initialization. */
9863         strcpy(mp->name, name);
9864         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9865         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9866         if (status) {
9867                 free(mp);
9868                 CHECK(0, EINVAL);
9869         }
9870
9871         /* Node add to tailq. */
9872         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9873
9874         return 0;
9875 }
9876
9877 int
9878 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9879                                  const char *name)
9880 {
9881         struct meter_profile *mp;
9882
9883         CHECK(p, EINVAL);
9884         CHECK_NAME(name, EINVAL);
9885
9886         mp = meter_profile_find(p, name);
9887         CHECK(mp, EINVAL);
9888         CHECK(!mp->n_users, EBUSY);
9889
9890         /* Remove node from tailq. */
9891         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9892         free(mp);
9893
9894         return 0;
9895 }
9896
9897 int
9898 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9899                         const char *metarray_name,
9900                         uint32_t metarray_index)
9901 {
9902         struct meter_profile *mp_old;
9903         struct metarray *metarray;
9904         struct metarray_runtime *metarray_runtime;
9905         struct meter *m;
9906
9907         CHECK(p, EINVAL);
9908         CHECK_NAME(metarray_name, EINVAL);
9909
9910         metarray = metarray_find(p, metarray_name);
9911         CHECK(metarray, EINVAL);
9912         CHECK(metarray_index < metarray->size, EINVAL);
9913
9914         metarray_runtime = &p->metarray_runtime[metarray->id];
9915         m = &metarray_runtime->metarray[metarray_index];
9916         mp_old = m->profile;
9917
9918         meter_init(m);
9919
9920         mp_old->n_users--;
9921
9922         return 0;
9923 }
9924
9925 int
9926 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9927                       const char *metarray_name,
9928                       uint32_t metarray_index,
9929                       const char *profile_name)
9930 {
9931         struct meter_profile *mp, *mp_old;
9932         struct metarray *metarray;
9933         struct metarray_runtime *metarray_runtime;
9934         struct meter *m;
9935
9936         CHECK(p, EINVAL);
9937         CHECK_NAME(metarray_name, EINVAL);
9938
9939         metarray = metarray_find(p, metarray_name);
9940         CHECK(metarray, EINVAL);
9941         CHECK(metarray_index < metarray->size, EINVAL);
9942
9943         mp = meter_profile_find(p, profile_name);
9944         CHECK(mp, EINVAL);
9945
9946         metarray_runtime = &p->metarray_runtime[metarray->id];
9947         m = &metarray_runtime->metarray[metarray_index];
9948         mp_old = m->profile;
9949
9950         memset(m, 0, sizeof(struct meter));
9951         rte_meter_trtcm_config(&m->m, &mp->profile);
9952         m->profile = mp;
9953         m->color_mask = RTE_COLORS;
9954
9955         mp->n_users++;
9956         mp_old->n_users--;
9957
9958         return 0;
9959 }
9960
9961 int
9962 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9963                              const char *metarray_name,
9964                              uint32_t metarray_index,
9965                              struct rte_swx_ctl_meter_stats *stats)
9966 {
9967         struct metarray *metarray;
9968         struct metarray_runtime *metarray_runtime;
9969         struct meter *m;
9970
9971         CHECK(p, EINVAL);
9972         CHECK_NAME(metarray_name, EINVAL);
9973
9974         metarray = metarray_find(p, metarray_name);
9975         CHECK(metarray, EINVAL);
9976         CHECK(metarray_index < metarray->size, EINVAL);
9977
9978         CHECK(stats, EINVAL);
9979
9980         metarray_runtime = &p->metarray_runtime[metarray->id];
9981         m = &metarray_runtime->metarray[metarray_index];
9982
9983         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9984         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9985
9986         return 0;
9987 }