lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <inttypes.h>
   9 #include <sys/queue.h>
  10 #include <arpa/inet.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_prefetch.h>
  14 #include <rte_byteorder.h>
  15 #include <rte_cycles.h>
  16 #include <rte_meter.h>
  17
  18 #include "rte_swx_pipeline.h"
  19 #include "rte_swx_ctl.h"
  20
  21 #define CHECK(condition, err_code)                                             \
  22 do {                                                                           \
  23         if (!(condition))                                                      \
  24                 return -(err_code);                                            \
  25 } while (0)
  26
  27 #define CHECK_NAME(name, err_code)                                             \
  28         CHECK((name) &&                                                        \
  29               (name)[0] &&                                                     \
  30               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  31               err_code)
  32
  33 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  34         CHECK((instr) &&                                                       \
  35               (instr)[0] &&                                                    \
  36               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  37                RTE_SWX_INSTRUCTION_SIZE),                                      \
  38               err_code)
  39
  40 #ifndef TRACE_LEVEL
  41 #define TRACE_LEVEL 0
  42 #endif
  43
  44 #if TRACE_LEVEL
  45 #define TRACE(...) printf(__VA_ARGS__)
  46 #else
  47 #define TRACE(...)
  48 #endif
  49
  50 /*
  51  * Environment.
  52  */
  53 #define ntoh64(x) rte_be_to_cpu_64(x)
  54 #define hton64(x) rte_cpu_to_be_64(x)
  55
  56 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  57
  58 #include <rte_malloc.h>
  59
  60 static void *
  61 env_malloc(size_t size, size_t alignment, int numa_node)
  62 {
  63         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  64 }
  65
  66 static void
  67 env_free(void *start, size_t size __rte_unused)
  68 {
  69         rte_free(start);
  70 }
  71
  72 #else
  73
  74 #include <numa.h>
  75
  76 static void *
  77 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  78 {
  79         void *start;
  80
  81         if (numa_available() == -1)
  82                 return NULL;
  83
  84         start = numa_alloc_onnode(size, numa_node);
  85         if (!start)
  86                 return NULL;
  87
  88         memset(start, 0, size);
  89         return start;
  90 }
  91
  92 static void
  93 env_free(void *start, size_t size)
  94 {
  95         if (numa_available() == -1)
  96                 return;
  97
  98         numa_free(start, size);
  99 }
 100
 101 #endif
 102
 103 /*
 104  * Struct.
 105  */
 106 struct field {
 107         char name[RTE_SWX_NAME_SIZE];
 108         uint32_t n_bits;
 109         uint32_t offset;
 110 };
 111
 112 struct struct_type {
 113         TAILQ_ENTRY(struct_type) node;
 114         char name[RTE_SWX_NAME_SIZE];
 115         struct field *fields;
 116         uint32_t n_fields;
 117         uint32_t n_bits;
 118 };
 119
 120 TAILQ_HEAD(struct_type_tailq, struct_type);
 121
 122 /*
 123  * Input port.
 124  */
 125 struct port_in_type {
 126         TAILQ_ENTRY(port_in_type) node;
 127         char name[RTE_SWX_NAME_SIZE];
 128         struct rte_swx_port_in_ops ops;
 129 };
 130
 131 TAILQ_HEAD(port_in_type_tailq, port_in_type);
 132
 133 struct port_in {
 134         TAILQ_ENTRY(port_in) node;
 135         struct port_in_type *type;
 136         void *obj;
 137         uint32_t id;
 138 };
 139
 140 TAILQ_HEAD(port_in_tailq, port_in);
 141
 142 struct port_in_runtime {
 143         rte_swx_port_in_pkt_rx_t pkt_rx;
 144         void *obj;
 145 };
 146
 147 /*
 148  * Output port.
 149  */
 150 struct port_out_type {
 151         TAILQ_ENTRY(port_out_type) node;
 152         char name[RTE_SWX_NAME_SIZE];
 153         struct rte_swx_port_out_ops ops;
 154 };
 155
 156 TAILQ_HEAD(port_out_type_tailq, port_out_type);
 157
 158 struct port_out {
 159         TAILQ_ENTRY(port_out) node;
 160         struct port_out_type *type;
 161         void *obj;
 162         uint32_t id;
 163 };
 164
 165 TAILQ_HEAD(port_out_tailq, port_out);
 166
 167 struct port_out_runtime {
 168         rte_swx_port_out_pkt_tx_t pkt_tx;
 169         rte_swx_port_out_flush_t flush;
 170         void *obj;
 171 };
 172
 173 /*
 174  * Extern object.
 175  */
 176 struct extern_type_member_func {
 177         TAILQ_ENTRY(extern_type_member_func) node;
 178         char name[RTE_SWX_NAME_SIZE];
 179         rte_swx_extern_type_member_func_t func;
 180         uint32_t id;
 181 };
 182
 183 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 184
 185 struct extern_type {
 186         TAILQ_ENTRY(extern_type) node;
 187         char name[RTE_SWX_NAME_SIZE];
 188         struct struct_type *mailbox_struct_type;
 189         rte_swx_extern_type_constructor_t constructor;
 190         rte_swx_extern_type_destructor_t destructor;
 191         struct extern_type_member_func_tailq funcs;
 192         uint32_t n_funcs;
 193 };
 194
 195 TAILQ_HEAD(extern_type_tailq, extern_type);
 196
 197 struct extern_obj {
 198         TAILQ_ENTRY(extern_obj) node;
 199         char name[RTE_SWX_NAME_SIZE];
 200         struct extern_type *type;
 201         void *obj;
 202         uint32_t struct_id;
 203         uint32_t id;
 204 };
 205
 206 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 207
 208 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 209 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 210 #endif
 211
 212 struct extern_obj_runtime {
 213         void *obj;
 214         uint8_t *mailbox;
 215         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 216 };
 217
 218 /*
 219  * Extern function.
 220  */
 221 struct extern_func {
 222         TAILQ_ENTRY(extern_func) node;
 223         char name[RTE_SWX_NAME_SIZE];
 224         struct struct_type *mailbox_struct_type;
 225         rte_swx_extern_func_t func;
 226         uint32_t struct_id;
 227         uint32_t id;
 228 };
 229
 230 TAILQ_HEAD(extern_func_tailq, extern_func);
 231
 232 struct extern_func_runtime {
 233         uint8_t *mailbox;
 234         rte_swx_extern_func_t func;
 235 };
 236
 237 /*
 238  * Header.
 239  */
 240 struct header {
 241         TAILQ_ENTRY(header) node;
 242         char name[RTE_SWX_NAME_SIZE];
 243         struct struct_type *st;
 244         uint32_t struct_id;
 245         uint32_t id;
 246 };
 247
 248 TAILQ_HEAD(header_tailq, header);
 249
 250 struct header_runtime {
 251         uint8_t *ptr0;
 252 };
 253
 254 struct header_out_runtime {
 255         uint8_t *ptr0;
 256         uint8_t *ptr;
 257         uint32_t n_bytes;
 258 };
 259
 260 /*
 261  * Instruction.
 262  */
 263
 264 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 265  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 266  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 267  * when transferred to packet meta-data and in NBO when transferred to packet
 268  * headers.
 269  */
 270
 271 /* Notation conventions:
 272  *    -Header field: H = h.header.field (dst/src)
 273  *    -Meta-data field: M = m.field (dst/src)
 274  *    -Extern object mailbox field: E = e.field (dst/src)
 275  *    -Extern function mailbox field: F = f.field (dst/src)
 276  *    -Table action data field: T = t.field (src only)
 277  *    -Immediate value: I = 32-bit unsigned value (src only)
 278  */
 279
 280 enum instruction_type {
 281         /* rx m.port_in */
 282         INSTR_RX,
 283
 284         /* tx m.port_out */
 285         INSTR_TX,
 286
 287         /* extract h.header */
 288         INSTR_HDR_EXTRACT,
 289         INSTR_HDR_EXTRACT2,
 290         INSTR_HDR_EXTRACT3,
 291         INSTR_HDR_EXTRACT4,
 292         INSTR_HDR_EXTRACT5,
 293         INSTR_HDR_EXTRACT6,
 294         INSTR_HDR_EXTRACT7,
 295         INSTR_HDR_EXTRACT8,
 296
 297         /* emit h.header */
 298         INSTR_HDR_EMIT,
 299         INSTR_HDR_EMIT_TX,
 300         INSTR_HDR_EMIT2_TX,
 301         INSTR_HDR_EMIT3_TX,
 302         INSTR_HDR_EMIT4_TX,
 303         INSTR_HDR_EMIT5_TX,
 304         INSTR_HDR_EMIT6_TX,
 305         INSTR_HDR_EMIT7_TX,
 306         INSTR_HDR_EMIT8_TX,
 307
 308         /* validate h.header */
 309         INSTR_HDR_VALIDATE,
 310
 311         /* invalidate h.header */
 312         INSTR_HDR_INVALIDATE,
 313
 314         /* mov dst src
 315          * dst = src
 316          * dst = HMEF, src = HMEFTI
 317          */
 318         INSTR_MOV,   /* dst = MEF, src = MEFT */
 319         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 320         INSTR_MOV_I, /* dst = HMEF, src = I */
 321
 322         /* dma h.header t.field
 323          * memcpy(h.header, t.field, sizeof(h.header))
 324          */
 325         INSTR_DMA_HT,
 326         INSTR_DMA_HT2,
 327         INSTR_DMA_HT3,
 328         INSTR_DMA_HT4,
 329         INSTR_DMA_HT5,
 330         INSTR_DMA_HT6,
 331         INSTR_DMA_HT7,
 332         INSTR_DMA_HT8,
 333
 334         /* add dst src
 335          * dst += src
 336          * dst = HMEF, src = HMEFTI
 337          */
 338         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 339         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 340         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 341         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 342         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 343         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 344
 345         /* sub dst src
 346          * dst -= src
 347          * dst = HMEF, src = HMEFTI
 348          */
 349         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 350         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 351         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 352         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 353         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 354         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 355
 356         /* ckadd dst src
 357          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 358          * dst = H, src = {H, h.header}
 359          */
 360         INSTR_ALU_CKADD_FIELD,    /* src = H */
 361         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 362         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 363
 364         /* cksub dst src
 365          * dst = dst '- src
 366          * dst = H, src = H
 367          */
 368         INSTR_ALU_CKSUB_FIELD,
 369
 370         /* and dst src
 371          * dst &= src
 372          * dst = HMEF, src = HMEFTI
 373          */
 374         INSTR_ALU_AND,   /* dst = MEF, src = MEFT */
 375         INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 376         INSTR_ALU_AND_I, /* dst = HMEF, src = I */
 377
 378         /* or dst src
 379          * dst |= src
 380          * dst = HMEF, src = HMEFTI
 381          */
 382         INSTR_ALU_OR,   /* dst = MEF, src = MEFT */
 383         INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 384         INSTR_ALU_OR_I, /* dst = HMEF, src = I */
 385
 386         /* xor dst src
 387          * dst ^= src
 388          * dst = HMEF, src = HMEFTI
 389          */
 390         INSTR_ALU_XOR,   /* dst = MEF, src = MEFT */
 391         INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 392         INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
 393
 394         /* shl dst src
 395          * dst <<= src
 396          * dst = HMEF, src = HMEFTI
 397          */
 398         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 399         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 400         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 401         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 402         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 403         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 404
 405         /* shr dst src
 406          * dst >>= src
 407          * dst = HMEF, src = HMEFTI
 408          */
 409         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 410         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 411         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 412         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 413         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 414         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 415
 416         /* regprefetch REGARRAY index
 417          * prefetch REGARRAY[index]
 418          * index = HMEFTI
 419          */
 420         INSTR_REGPREFETCH_RH, /* index = H */
 421         INSTR_REGPREFETCH_RM, /* index = MEFT */
 422         INSTR_REGPREFETCH_RI, /* index = I */
 423
 424         /* regrd dst REGARRAY index
 425          * dst = REGARRAY[index]
 426          * dst = HMEF, index = HMEFTI
 427          */
 428         INSTR_REGRD_HRH, /* dst = H, index = H */
 429         INSTR_REGRD_HRM, /* dst = H, index = MEFT */
 430         INSTR_REGRD_HRI, /* dst = H, index = I */
 431         INSTR_REGRD_MRH, /* dst = MEF, index = H */
 432         INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
 433         INSTR_REGRD_MRI, /* dst = MEF, index = I */
 434
 435         /* regwr REGARRAY index src
 436          * REGARRAY[index] = src
 437          * index = HMEFTI, src = HMEFTI
 438          */
 439         INSTR_REGWR_RHH, /* index = H, src = H */
 440         INSTR_REGWR_RHM, /* index = H, src = MEFT */
 441         INSTR_REGWR_RHI, /* index = H, src = I */
 442         INSTR_REGWR_RMH, /* index = MEFT, src = H */
 443         INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
 444         INSTR_REGWR_RMI, /* index = MEFT, src = I */
 445         INSTR_REGWR_RIH, /* index = I, src = H */
 446         INSTR_REGWR_RIM, /* index = I, src = MEFT */
 447         INSTR_REGWR_RII, /* index = I, src = I */
 448
 449         /* regadd REGARRAY index src
 450          * REGARRAY[index] += src
 451          * index = HMEFTI, src = HMEFTI
 452          */
 453         INSTR_REGADD_RHH, /* index = H, src = H */
 454         INSTR_REGADD_RHM, /* index = H, src = MEFT */
 455         INSTR_REGADD_RHI, /* index = H, src = I */
 456         INSTR_REGADD_RMH, /* index = MEFT, src = H */
 457         INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
 458         INSTR_REGADD_RMI, /* index = MEFT, src = I */
 459         INSTR_REGADD_RIH, /* index = I, src = H */
 460         INSTR_REGADD_RIM, /* index = I, src = MEFT */
 461         INSTR_REGADD_RII, /* index = I, src = I */
 462
 463         /* metprefetch METARRAY index
 464          * prefetch METARRAY[index]
 465          * index = HMEFTI
 466          */
 467         INSTR_METPREFETCH_H, /* index = H */
 468         INSTR_METPREFETCH_M, /* index = MEFT */
 469         INSTR_METPREFETCH_I, /* index = I */
 470
 471         /* meter METARRAY index length color_in color_out
 472          * color_out = meter(METARRAY[index], length, color_in)
 473          * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
 474          */
 475         INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
 476         INSTR_METER_HHI, /* index = H, length = H, color_in = I */
 477         INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
 478         INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
 479         INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
 480         INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
 481         INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
 482         INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
 483         INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
 484         INSTR_METER_IHI, /* index = I, length = H, color_in = I */
 485         INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
 486         INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
 487
 488         /* table TABLE */
 489         INSTR_TABLE,
 490
 491         /* extern e.obj.func */
 492         INSTR_EXTERN_OBJ,
 493
 494         /* extern f.func */
 495         INSTR_EXTERN_FUNC,
 496
 497         /* jmp LABEL
 498          * Unconditional jump
 499          */
 500         INSTR_JMP,
 501
 502         /* jmpv LABEL h.header
 503          * Jump if header is valid
 504          */
 505         INSTR_JMP_VALID,
 506
 507         /* jmpnv LABEL h.header
 508          * Jump if header is invalid
 509          */
 510         INSTR_JMP_INVALID,
 511
 512         /* jmph LABEL
 513          * Jump if table lookup hit
 514          */
 515         INSTR_JMP_HIT,
 516
 517         /* jmpnh LABEL
 518          * Jump if table lookup miss
 519          */
 520         INSTR_JMP_MISS,
 521
 522         /* jmpa LABEL ACTION
 523          * Jump if action run
 524          */
 525         INSTR_JMP_ACTION_HIT,
 526
 527         /* jmpna LABEL ACTION
 528          * Jump if action not run
 529          */
 530         INSTR_JMP_ACTION_MISS,
 531
 532         /* jmpeq LABEL a b
 533          * Jump is a is equal to b
 534          * a = HMEFT, b = HMEFTI
 535          */
 536         INSTR_JMP_EQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
 537         INSTR_JMP_EQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
 538         INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 539
 540         /* jmpneq LABEL a b
 541          * Jump is a is not equal to b
 542          * a = HMEFT, b = HMEFTI
 543          */
 544         INSTR_JMP_NEQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
 545         INSTR_JMP_NEQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
 546         INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 547
 548         /* jmplt LABEL a b
 549          * Jump if a is less than b
 550          * a = HMEFT, b = HMEFTI
 551          */
 552         INSTR_JMP_LT,    /* a = MEF, b = MEF */
 553         INSTR_JMP_LT_MH, /* a = MEF, b = H */
 554         INSTR_JMP_LT_HM, /* a = H, b = MEF */
 555         INSTR_JMP_LT_HH, /* a = H, b = H */
 556         INSTR_JMP_LT_MI, /* a = MEF, b = I */
 557         INSTR_JMP_LT_HI, /* a = H, b = I */
 558
 559         /* jmpgt LABEL a b
 560          * Jump if a is greater than b
 561          * a = HMEFT, b = HMEFTI
 562          */
 563         INSTR_JMP_GT,    /* a = MEF, b = MEF */
 564         INSTR_JMP_GT_MH, /* a = MEF, b = H */
 565         INSTR_JMP_GT_HM, /* a = H, b = MEF */
 566         INSTR_JMP_GT_HH, /* a = H, b = H */
 567         INSTR_JMP_GT_MI, /* a = MEF, b = I */
 568         INSTR_JMP_GT_HI, /* a = H, b = I */
 569
 570         /* return
 571          * Return from action
 572          */
 573         INSTR_RETURN,
 574 };
 575
 576 struct instr_operand {
 577         uint8_t struct_id;
 578         uint8_t n_bits;
 579         uint8_t offset;
 580         uint8_t pad;
 581 };
 582
 583 struct instr_io {
 584         struct {
 585                 uint8_t offset;
 586                 uint8_t n_bits;
 587                 uint8_t pad[2];
 588         } io;
 589
 590         struct {
 591                 uint8_t header_id[8];
 592                 uint8_t struct_id[8];
 593                 uint8_t n_bytes[8];
 594         } hdr;
 595 };
 596
 597 struct instr_hdr_validity {
 598         uint8_t header_id;
 599 };
 600
 601 struct instr_table {
 602         uint8_t table_id;
 603 };
 604
 605 struct instr_extern_obj {
 606         uint8_t ext_obj_id;
 607         uint8_t func_id;
 608 };
 609
 610 struct instr_extern_func {
 611         uint8_t ext_func_id;
 612 };
 613
 614 struct instr_dst_src {
 615         struct instr_operand dst;
 616         union {
 617                 struct instr_operand src;
 618                 uint64_t src_val;
 619         };
 620 };
 621
 622 struct instr_regarray {
 623         uint8_t regarray_id;
 624         uint8_t pad[3];
 625
 626         union {
 627                 struct instr_operand idx;
 628                 uint32_t idx_val;
 629         };
 630
 631         union {
 632                 struct instr_operand dstsrc;
 633                 uint64_t dstsrc_val;
 634         };
 635 };
 636
 637 struct instr_meter {
 638         uint8_t metarray_id;
 639         uint8_t pad[3];
 640
 641         union {
 642                 struct instr_operand idx;
 643                 uint32_t idx_val;
 644         };
 645
 646         struct instr_operand length;
 647
 648         union {
 649                 struct instr_operand color_in;
 650                 uint32_t color_in_val;
 651         };
 652
 653         struct instr_operand color_out;
 654 };
 655
 656 struct instr_dma {
 657         struct {
 658                 uint8_t header_id[8];
 659                 uint8_t struct_id[8];
 660         } dst;
 661
 662         struct {
 663                 uint8_t offset[8];
 664         } src;
 665
 666         uint16_t n_bytes[8];
 667 };
 668
 669 struct instr_jmp {
 670         struct instruction *ip;
 671
 672         union {
 673                 struct instr_operand a;
 674                 uint8_t header_id;
 675                 uint8_t action_id;
 676         };
 677
 678         union {
 679                 struct instr_operand b;
 680                 uint64_t b_val;
 681         };
 682 };
 683
 684 struct instruction {
 685         enum instruction_type type;
 686         union {
 687                 struct instr_io io;
 688                 struct instr_hdr_validity valid;
 689                 struct instr_dst_src mov;
 690                 struct instr_regarray regarray;
 691                 struct instr_meter meter;
 692                 struct instr_dma dma;
 693                 struct instr_dst_src alu;
 694                 struct instr_table table;
 695                 struct instr_extern_obj ext_obj;
 696                 struct instr_extern_func ext_func;
 697                 struct instr_jmp jmp;
 698         };
 699 };
 700
 701 struct instruction_data {
 702         char label[RTE_SWX_NAME_SIZE];
 703         char jmp_label[RTE_SWX_NAME_SIZE];
 704         uint32_t n_users; /* user = jmp instruction to this instruction. */
 705         int invalid;
 706 };
 707
 708 /*
 709  * Action.
 710  */
 711 struct action {
 712         TAILQ_ENTRY(action) node;
 713         char name[RTE_SWX_NAME_SIZE];
 714         struct struct_type *st;
 715         struct instruction *instructions;
 716         uint32_t n_instructions;
 717         uint32_t id;
 718 };
 719
 720 TAILQ_HEAD(action_tailq, action);
 721
 722 /*
 723  * Table.
 724  */
 725 struct table_type {
 726         TAILQ_ENTRY(table_type) node;
 727         char name[RTE_SWX_NAME_SIZE];
 728         enum rte_swx_table_match_type match_type;
 729         struct rte_swx_table_ops ops;
 730 };
 731
 732 TAILQ_HEAD(table_type_tailq, table_type);
 733
 734 struct match_field {
 735         enum rte_swx_table_match_type match_type;
 736         struct field *field;
 737 };
 738
 739 struct table {
 740         TAILQ_ENTRY(table) node;
 741         char name[RTE_SWX_NAME_SIZE];
 742         char args[RTE_SWX_NAME_SIZE];
 743         struct table_type *type; /* NULL when n_fields == 0. */
 744
 745         /* Match. */
 746         struct match_field *fields;
 747         uint32_t n_fields;
 748         struct header *header; /* Only valid when n_fields > 0. */
 749
 750         /* Action. */
 751         struct action **actions;
 752         struct action *default_action;
 753         uint8_t *default_action_data;
 754         uint32_t n_actions;
 755         int default_action_is_const;
 756         uint32_t action_data_size_max;
 757
 758         uint32_t size;
 759         uint32_t id;
 760 };
 761
 762 TAILQ_HEAD(table_tailq, table);
 763
 764 struct table_runtime {
 765         rte_swx_table_lookup_t func;
 766         void *mailbox;
 767         uint8_t **key;
 768 };
 769
 770 /*
 771  * Register array.
 772  */
 773 struct regarray {
 774         TAILQ_ENTRY(regarray) node;
 775         char name[RTE_SWX_NAME_SIZE];
 776         uint64_t init_val;
 777         uint32_t size;
 778         uint32_t id;
 779 };
 780
 781 TAILQ_HEAD(regarray_tailq, regarray);
 782
 783 struct regarray_runtime {
 784         uint64_t *regarray;
 785         uint32_t size_mask;
 786 };
 787
 788 /*
 789  * Meter array.
 790  */
 791 struct meter_profile {
 792         TAILQ_ENTRY(meter_profile) node;
 793         char name[RTE_SWX_NAME_SIZE];
 794         struct rte_meter_trtcm_params params;
 795         struct rte_meter_trtcm_profile profile;
 796         uint32_t n_users;
 797 };
 798
 799 TAILQ_HEAD(meter_profile_tailq, meter_profile);
 800
 801 struct metarray {
 802         TAILQ_ENTRY(metarray) node;
 803         char name[RTE_SWX_NAME_SIZE];
 804         uint32_t size;
 805         uint32_t id;
 806 };
 807
 808 TAILQ_HEAD(metarray_tailq, metarray);
 809
 810 struct meter {
 811         struct rte_meter_trtcm m;
 812         struct meter_profile *profile;
 813         enum rte_color color_mask;
 814         uint8_t pad[20];
 815
 816         uint64_t n_pkts[RTE_COLORS];
 817         uint64_t n_bytes[RTE_COLORS];
 818 };
 819
 820 struct metarray_runtime {
 821         struct meter *metarray;
 822         uint32_t size_mask;
 823 };
 824
 825 /*
 826  * Pipeline.
 827  */
 828 struct thread {
 829         /* Packet. */
 830         struct rte_swx_pkt pkt;
 831         uint8_t *ptr;
 832
 833         /* Structures. */
 834         uint8_t **structs;
 835
 836         /* Packet headers. */
 837         struct header_runtime *headers; /* Extracted or generated headers. */
 838         struct header_out_runtime *headers_out; /* Emitted headers. */
 839         uint8_t *header_storage;
 840         uint8_t *header_out_storage;
 841         uint64_t valid_headers;
 842         uint32_t n_headers_out;
 843
 844         /* Packet meta-data. */
 845         uint8_t *metadata;
 846
 847         /* Tables. */
 848         struct table_runtime *tables;
 849         struct rte_swx_table_state *table_state;
 850         uint64_t action_id;
 851         int hit; /* 0 = Miss, 1 = Hit. */
 852
 853         /* Extern objects and functions. */
 854         struct extern_obj_runtime *extern_objs;
 855         struct extern_func_runtime *extern_funcs;
 856
 857         /* Instructions. */
 858         struct instruction *ip;
 859         struct instruction *ret;
 860 };
 861
 862 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 863 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 864 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 865
 866 #define HEADER_VALID(thread, header_id) \
 867         MASK64_BIT_GET((thread)->valid_headers, header_id)
 868
 869 #define ALU(thread, ip, operator)  \
 870 {                                                                              \
 871         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 872         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 873         uint64_t dst64 = *dst64_ptr;                                           \
 874         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 875         uint64_t dst = dst64 & dst64_mask;                                     \
 876                                                                                \
 877         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 878         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 879         uint64_t src64 = *src64_ptr;                                           \
 880         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 881         uint64_t src = src64 & src64_mask;                                     \
 882                                                                                \
 883         uint64_t result = dst operator src;                                    \
 884                                                                                \
 885         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 886 }
 887
 888 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 889
 890 #define ALU_S(thread, ip, operator)  \
 891 {                                                                              \
 892         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 893         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 894         uint64_t dst64 = *dst64_ptr;                                           \
 895         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 896         uint64_t dst = dst64 & dst64_mask;                                     \
 897                                                                                \
 898         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 899         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 900         uint64_t src64 = *src64_ptr;                                           \
 901         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 902                                                                                \
 903         uint64_t result = dst operator src;                                    \
 904                                                                                \
 905         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 906 }
 907
 908 #define ALU_MH ALU_S
 909
 910 #define ALU_HM(thread, ip, operator)  \
 911 {                                                                              \
 912         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 913         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 914         uint64_t dst64 = *dst64_ptr;                                           \
 915         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 916         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 917                                                                                \
 918         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 919         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 920         uint64_t src64 = *src64_ptr;                                           \
 921         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 922         uint64_t src = src64 & src64_mask;                                     \
 923                                                                                \
 924         uint64_t result = dst operator src;                                    \
 925         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 926                                                                                \
 927         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 928 }
 929
 930 #define ALU_HH(thread, ip, operator)  \
 931 {                                                                              \
 932         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 933         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 934         uint64_t dst64 = *dst64_ptr;                                           \
 935         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 936         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 937                                                                                \
 938         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 939         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 940         uint64_t src64 = *src64_ptr;                                           \
 941         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 942                                                                                \
 943         uint64_t result = dst operator src;                                    \
 944         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 945                                                                                \
 946         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 947 }
 948
 949 #else
 950
 951 #define ALU_S ALU
 952 #define ALU_MH ALU
 953 #define ALU_HM ALU
 954 #define ALU_HH ALU
 955
 956 #endif
 957
 958 #define ALU_I(thread, ip, operator)  \
 959 {                                                                              \
 960         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 961         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 962         uint64_t dst64 = *dst64_ptr;                                           \
 963         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 964         uint64_t dst = dst64 & dst64_mask;                                     \
 965                                                                                \
 966         uint64_t src = (ip)->alu.src_val;                                      \
 967                                                                                \
 968         uint64_t result = dst operator src;                                    \
 969                                                                                \
 970         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 971 }
 972
 973 #define ALU_MI ALU_I
 974
 975 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 976
 977 #define ALU_HI(thread, ip, operator)  \
 978 {                                                                              \
 979         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 980         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 981         uint64_t dst64 = *dst64_ptr;                                           \
 982         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 983         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 984                                                                                \
 985         uint64_t src = (ip)->alu.src_val;                                      \
 986                                                                                \
 987         uint64_t result = dst operator src;                                    \
 988         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 989                                                                                \
 990         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 991 }
 992
 993 #else
 994
 995 #define ALU_HI ALU_I
 996
 997 #endif
 998
 999 #define MOV(thread, ip)  \
1000 {                                                                              \
1001         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1002         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1003         uint64_t dst64 = *dst64_ptr;                                           \
1004         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1005                                                                                \
1006         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1007         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1008         uint64_t src64 = *src64_ptr;                                           \
1009         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1010         uint64_t src = src64 & src64_mask;                                     \
1011                                                                                \
1012         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1013 }
1014
1015 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1016
1017 #define MOV_S(thread, ip)  \
1018 {                                                                              \
1019         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1020         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1021         uint64_t dst64 = *dst64_ptr;                                           \
1022         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1023                                                                                \
1024         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1025         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1026         uint64_t src64 = *src64_ptr;                                           \
1027         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
1028                                                                                \
1029         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1030 }
1031
1032 #else
1033
1034 #define MOV_S MOV
1035
1036 #endif
1037
1038 #define MOV_I(thread, ip)  \
1039 {                                                                              \
1040         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1041         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1042         uint64_t dst64 = *dst64_ptr;                                           \
1043         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1044                                                                                \
1045         uint64_t src = (ip)->mov.src_val;                                      \
1046                                                                                \
1047         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1048 }
1049
1050 #define JMP_CMP(thread, ip, operator)  \
1051 {                                                                              \
1052         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1053         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1054         uint64_t a64 = *a64_ptr;                                               \
1055         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1056         uint64_t a = a64 & a64_mask;                                           \
1057                                                                                \
1058         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1059         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1060         uint64_t b64 = *b64_ptr;                                               \
1061         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1062         uint64_t b = b64 & b64_mask;                                           \
1063                                                                                \
1064         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1065 }
1066
1067 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1068
1069 #define JMP_CMP_S(thread, ip, operator)  \
1070 {                                                                              \
1071         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1072         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1073         uint64_t a64 = *a64_ptr;                                               \
1074         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1075         uint64_t a = a64 & a64_mask;                                           \
1076                                                                                \
1077         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1078         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1079         uint64_t b64 = *b64_ptr;                                               \
1080         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1081                                                                                \
1082         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1083 }
1084
1085 #define JMP_CMP_MH JMP_CMP_S
1086
1087 #define JMP_CMP_HM(thread, ip, operator)  \
1088 {                                                                              \
1089         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1090         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1091         uint64_t a64 = *a64_ptr;                                               \
1092         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1093                                                                                \
1094         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1095         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1096         uint64_t b64 = *b64_ptr;                                               \
1097         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1098         uint64_t b = b64 & b64_mask;                                           \
1099                                                                                \
1100         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1101 }
1102
1103 #define JMP_CMP_HH(thread, ip, operator)  \
1104 {                                                                              \
1105         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1106         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1107         uint64_t a64 = *a64_ptr;                                               \
1108         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1109                                                                                \
1110         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1111         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1112         uint64_t b64 = *b64_ptr;                                               \
1113         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1114                                                                                \
1115         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1116 }
1117
1118 #else
1119
1120 #define JMP_CMP_S JMP_CMP
1121 #define JMP_CMP_MH JMP_CMP
1122 #define JMP_CMP_HM JMP_CMP
1123 #define JMP_CMP_HH JMP_CMP
1124
1125 #endif
1126
1127 #define JMP_CMP_I(thread, ip, operator)  \
1128 {                                                                              \
1129         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1130         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1131         uint64_t a64 = *a64_ptr;                                               \
1132         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1133         uint64_t a = a64 & a64_mask;                                           \
1134                                                                                \
1135         uint64_t b = (ip)->jmp.b_val;                                          \
1136                                                                                \
1137         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1138 }
1139
1140 #define JMP_CMP_MI JMP_CMP_I
1141
1142 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1143
1144 #define JMP_CMP_HI(thread, ip, operator)  \
1145 {                                                                              \
1146         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1147         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1148         uint64_t a64 = *a64_ptr;                                               \
1149         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1150                                                                                \
1151         uint64_t b = (ip)->jmp.b_val;                                          \
1152                                                                                \
1153         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1154 }
1155
1156 #else
1157
1158 #define JMP_CMP_HI JMP_CMP_I
1159
1160 #endif
1161
1162 #define METADATA_READ(thread, offset, n_bits)                                  \
1163 ({                                                                             \
1164         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1165         uint64_t m64 = *m64_ptr;                                               \
1166         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1167         (m64 & m64_mask);                                                      \
1168 })
1169
1170 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
1171 {                                                                              \
1172         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1173         uint64_t m64 = *m64_ptr;                                               \
1174         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1175                                                                                \
1176         uint64_t m_new = value;                                                \
1177                                                                                \
1178         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
1179 }
1180
1181 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1182 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1183 #endif
1184
1185 struct rte_swx_pipeline {
1186         struct struct_type_tailq struct_types;
1187         struct port_in_type_tailq port_in_types;
1188         struct port_in_tailq ports_in;
1189         struct port_out_type_tailq port_out_types;
1190         struct port_out_tailq ports_out;
1191         struct extern_type_tailq extern_types;
1192         struct extern_obj_tailq extern_objs;
1193         struct extern_func_tailq extern_funcs;
1194         struct header_tailq headers;
1195         struct struct_type *metadata_st;
1196         uint32_t metadata_struct_id;
1197         struct action_tailq actions;
1198         struct table_type_tailq table_types;
1199         struct table_tailq tables;
1200         struct regarray_tailq regarrays;
1201         struct meter_profile_tailq meter_profiles;
1202         struct metarray_tailq metarrays;
1203
1204         struct port_in_runtime *in;
1205         struct port_out_runtime *out;
1206         struct instruction **action_instructions;
1207         struct rte_swx_table_state *table_state;
1208         struct regarray_runtime *regarray_runtime;
1209         struct metarray_runtime *metarray_runtime;
1210         struct instruction *instructions;
1211         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1212
1213         uint32_t n_structs;
1214         uint32_t n_ports_in;
1215         uint32_t n_ports_out;
1216         uint32_t n_extern_objs;
1217         uint32_t n_extern_funcs;
1218         uint32_t n_actions;
1219         uint32_t n_tables;
1220         uint32_t n_regarrays;
1221         uint32_t n_metarrays;
1222         uint32_t n_headers;
1223         uint32_t thread_id;
1224         uint32_t port_id;
1225         uint32_t n_instructions;
1226         int build_done;
1227         int numa_node;
1228 };
1229
1230 /*
1231  * Struct.
1232  */
1233 static struct struct_type *
1234 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1235 {
1236         struct struct_type *elem;
1237
1238         TAILQ_FOREACH(elem, &p->struct_types, node)
1239                 if (strcmp(elem->name, name) == 0)
1240                         return elem;
1241
1242         return NULL;
1243 }
1244
1245 static struct field *
1246 struct_type_field_find(struct struct_type *st, const char *name)
1247 {
1248         uint32_t i;
1249
1250         for (i = 0; i < st->n_fields; i++) {
1251                 struct field *f = &st->fields[i];
1252
1253                 if (strcmp(f->name, name) == 0)
1254                         return f;
1255         }
1256
1257         return NULL;
1258 }
1259
1260 int
1261 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1262                                       const char *name,
1263                                       struct rte_swx_field_params *fields,
1264                                       uint32_t n_fields)
1265 {
1266         struct struct_type *st;
1267         uint32_t i;
1268
1269         CHECK(p, EINVAL);
1270         CHECK_NAME(name, EINVAL);
1271         CHECK(fields, EINVAL);
1272         CHECK(n_fields, EINVAL);
1273
1274         for (i = 0; i < n_fields; i++) {
1275                 struct rte_swx_field_params *f = &fields[i];
1276                 uint32_t j;
1277
1278                 CHECK_NAME(f->name, EINVAL);
1279                 CHECK(f->n_bits, EINVAL);
1280                 CHECK(f->n_bits <= 64, EINVAL);
1281                 CHECK((f->n_bits & 7) == 0, EINVAL);
1282
1283                 for (j = 0; j < i; j++) {
1284                         struct rte_swx_field_params *f_prev = &fields[j];
1285
1286                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
1287                 }
1288         }
1289
1290         CHECK(!struct_type_find(p, name), EEXIST);
1291
1292         /* Node allocation. */
1293         st = calloc(1, sizeof(struct struct_type));
1294         CHECK(st, ENOMEM);
1295
1296         st->fields = calloc(n_fields, sizeof(struct field));
1297         if (!st->fields) {
1298                 free(st);
1299                 CHECK(0, ENOMEM);
1300         }
1301
1302         /* Node initialization. */
1303         strcpy(st->name, name);
1304         for (i = 0; i < n_fields; i++) {
1305                 struct field *dst = &st->fields[i];
1306                 struct rte_swx_field_params *src = &fields[i];
1307
1308                 strcpy(dst->name, src->name);
1309                 dst->n_bits = src->n_bits;
1310                 dst->offset = st->n_bits;
1311
1312                 st->n_bits += src->n_bits;
1313         }
1314         st->n_fields = n_fields;
1315
1316         /* Node add to tailq. */
1317         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1318
1319         return 0;
1320 }
1321
1322 static int
1323 struct_build(struct rte_swx_pipeline *p)
1324 {
1325         uint32_t i;
1326
1327         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1328                 struct thread *t = &p->threads[i];
1329
1330                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1331                 CHECK(t->structs, ENOMEM);
1332         }
1333
1334         return 0;
1335 }
1336
1337 static void
1338 struct_build_free(struct rte_swx_pipeline *p)
1339 {
1340         uint32_t i;
1341
1342         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1343                 struct thread *t = &p->threads[i];
1344
1345                 free(t->structs);
1346                 t->structs = NULL;
1347         }
1348 }
1349
1350 static void
1351 struct_free(struct rte_swx_pipeline *p)
1352 {
1353         struct_build_free(p);
1354
1355         /* Struct types. */
1356         for ( ; ; ) {
1357                 struct struct_type *elem;
1358
1359                 elem = TAILQ_FIRST(&p->struct_types);
1360                 if (!elem)
1361                         break;
1362
1363                 TAILQ_REMOVE(&p->struct_types, elem, node);
1364                 free(elem->fields);
1365                 free(elem);
1366         }
1367 }
1368
1369 /*
1370  * Input port.
1371  */
1372 static struct port_in_type *
1373 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1374 {
1375         struct port_in_type *elem;
1376
1377         if (!name)
1378                 return NULL;
1379
1380         TAILQ_FOREACH(elem, &p->port_in_types, node)
1381                 if (strcmp(elem->name, name) == 0)
1382                         return elem;
1383
1384         return NULL;
1385 }
1386
1387 int
1388 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1389                                        const char *name,
1390                                        struct rte_swx_port_in_ops *ops)
1391 {
1392         struct port_in_type *elem;
1393
1394         CHECK(p, EINVAL);
1395         CHECK_NAME(name, EINVAL);
1396         CHECK(ops, EINVAL);
1397         CHECK(ops->create, EINVAL);
1398         CHECK(ops->free, EINVAL);
1399         CHECK(ops->pkt_rx, EINVAL);
1400         CHECK(ops->stats_read, EINVAL);
1401
1402         CHECK(!port_in_type_find(p, name), EEXIST);
1403
1404         /* Node allocation. */
1405         elem = calloc(1, sizeof(struct port_in_type));
1406         CHECK(elem, ENOMEM);
1407
1408         /* Node initialization. */
1409         strcpy(elem->name, name);
1410         memcpy(&elem->ops, ops, sizeof(*ops));
1411
1412         /* Node add to tailq. */
1413         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1414
1415         return 0;
1416 }
1417
1418 static struct port_in *
1419 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1420 {
1421         struct port_in *port;
1422
1423         TAILQ_FOREACH(port, &p->ports_in, node)
1424                 if (port->id == port_id)
1425                         return port;
1426
1427         return NULL;
1428 }
1429
1430 int
1431 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1432                                 uint32_t port_id,
1433                                 const char *port_type_name,
1434                                 void *args)
1435 {
1436         struct port_in_type *type = NULL;
1437         struct port_in *port = NULL;
1438         void *obj = NULL;
1439
1440         CHECK(p, EINVAL);
1441
1442         CHECK(!port_in_find(p, port_id), EINVAL);
1443
1444         CHECK_NAME(port_type_name, EINVAL);
1445         type = port_in_type_find(p, port_type_name);
1446         CHECK(type, EINVAL);
1447
1448         obj = type->ops.create(args);
1449         CHECK(obj, ENODEV);
1450
1451         /* Node allocation. */
1452         port = calloc(1, sizeof(struct port_in));
1453         CHECK(port, ENOMEM);
1454
1455         /* Node initialization. */
1456         port->type = type;
1457         port->obj = obj;
1458         port->id = port_id;
1459
1460         /* Node add to tailq. */
1461         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1462         if (p->n_ports_in < port_id + 1)
1463                 p->n_ports_in = port_id + 1;
1464
1465         return 0;
1466 }
1467
1468 static int
1469 port_in_build(struct rte_swx_pipeline *p)
1470 {
1471         struct port_in *port;
1472         uint32_t i;
1473
1474         CHECK(p->n_ports_in, EINVAL);
1475         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1476
1477         for (i = 0; i < p->n_ports_in; i++)
1478                 CHECK(port_in_find(p, i), EINVAL);
1479
1480         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1481         CHECK(p->in, ENOMEM);
1482
1483         TAILQ_FOREACH(port, &p->ports_in, node) {
1484                 struct port_in_runtime *in = &p->in[port->id];
1485
1486                 in->pkt_rx = port->type->ops.pkt_rx;
1487                 in->obj = port->obj;
1488         }
1489
1490         return 0;
1491 }
1492
1493 static void
1494 port_in_build_free(struct rte_swx_pipeline *p)
1495 {
1496         free(p->in);
1497         p->in = NULL;
1498 }
1499
1500 static void
1501 port_in_free(struct rte_swx_pipeline *p)
1502 {
1503         port_in_build_free(p);
1504
1505         /* Input ports. */
1506         for ( ; ; ) {
1507                 struct port_in *port;
1508
1509                 port = TAILQ_FIRST(&p->ports_in);
1510                 if (!port)
1511                         break;
1512
1513                 TAILQ_REMOVE(&p->ports_in, port, node);
1514                 port->type->ops.free(port->obj);
1515                 free(port);
1516         }
1517
1518         /* Input port types. */
1519         for ( ; ; ) {
1520                 struct port_in_type *elem;
1521
1522                 elem = TAILQ_FIRST(&p->port_in_types);
1523                 if (!elem)
1524                         break;
1525
1526                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1527                 free(elem);
1528         }
1529 }
1530
1531 /*
1532  * Output port.
1533  */
1534 static struct port_out_type *
1535 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1536 {
1537         struct port_out_type *elem;
1538
1539         if (!name)
1540                 return NULL;
1541
1542         TAILQ_FOREACH(elem, &p->port_out_types, node)
1543                 if (!strcmp(elem->name, name))
1544                         return elem;
1545
1546         return NULL;
1547 }
1548
1549 int
1550 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1551                                         const char *name,
1552                                         struct rte_swx_port_out_ops *ops)
1553 {
1554         struct port_out_type *elem;
1555
1556         CHECK(p, EINVAL);
1557         CHECK_NAME(name, EINVAL);
1558         CHECK(ops, EINVAL);
1559         CHECK(ops->create, EINVAL);
1560         CHECK(ops->free, EINVAL);
1561         CHECK(ops->pkt_tx, EINVAL);
1562         CHECK(ops->stats_read, EINVAL);
1563
1564         CHECK(!port_out_type_find(p, name), EEXIST);
1565
1566         /* Node allocation. */
1567         elem = calloc(1, sizeof(struct port_out_type));
1568         CHECK(elem, ENOMEM);
1569
1570         /* Node initialization. */
1571         strcpy(elem->name, name);
1572         memcpy(&elem->ops, ops, sizeof(*ops));
1573
1574         /* Node add to tailq. */
1575         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1576
1577         return 0;
1578 }
1579
1580 static struct port_out *
1581 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1582 {
1583         struct port_out *port;
1584
1585         TAILQ_FOREACH(port, &p->ports_out, node)
1586                 if (port->id == port_id)
1587                         return port;
1588
1589         return NULL;
1590 }
1591
1592 int
1593 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1594                                  uint32_t port_id,
1595                                  const char *port_type_name,
1596                                  void *args)
1597 {
1598         struct port_out_type *type = NULL;
1599         struct port_out *port = NULL;
1600         void *obj = NULL;
1601
1602         CHECK(p, EINVAL);
1603
1604         CHECK(!port_out_find(p, port_id), EINVAL);
1605
1606         CHECK_NAME(port_type_name, EINVAL);
1607         type = port_out_type_find(p, port_type_name);
1608         CHECK(type, EINVAL);
1609
1610         obj = type->ops.create(args);
1611         CHECK(obj, ENODEV);
1612
1613         /* Node allocation. */
1614         port = calloc(1, sizeof(struct port_out));
1615         CHECK(port, ENOMEM);
1616
1617         /* Node initialization. */
1618         port->type = type;
1619         port->obj = obj;
1620         port->id = port_id;
1621
1622         /* Node add to tailq. */
1623         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1624         if (p->n_ports_out < port_id + 1)
1625                 p->n_ports_out = port_id + 1;
1626
1627         return 0;
1628 }
1629
1630 static int
1631 port_out_build(struct rte_swx_pipeline *p)
1632 {
1633         struct port_out *port;
1634         uint32_t i;
1635
1636         CHECK(p->n_ports_out, EINVAL);
1637
1638         for (i = 0; i < p->n_ports_out; i++)
1639                 CHECK(port_out_find(p, i), EINVAL);
1640
1641         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1642         CHECK(p->out, ENOMEM);
1643
1644         TAILQ_FOREACH(port, &p->ports_out, node) {
1645                 struct port_out_runtime *out = &p->out[port->id];
1646
1647                 out->pkt_tx = port->type->ops.pkt_tx;
1648                 out->flush = port->type->ops.flush;
1649                 out->obj = port->obj;
1650         }
1651
1652         return 0;
1653 }
1654
1655 static void
1656 port_out_build_free(struct rte_swx_pipeline *p)
1657 {
1658         free(p->out);
1659         p->out = NULL;
1660 }
1661
1662 static void
1663 port_out_free(struct rte_swx_pipeline *p)
1664 {
1665         port_out_build_free(p);
1666
1667         /* Output ports. */
1668         for ( ; ; ) {
1669                 struct port_out *port;
1670
1671                 port = TAILQ_FIRST(&p->ports_out);
1672                 if (!port)
1673                         break;
1674
1675                 TAILQ_REMOVE(&p->ports_out, port, node);
1676                 port->type->ops.free(port->obj);
1677                 free(port);
1678         }
1679
1680         /* Output port types. */
1681         for ( ; ; ) {
1682                 struct port_out_type *elem;
1683
1684                 elem = TAILQ_FIRST(&p->port_out_types);
1685                 if (!elem)
1686                         break;
1687
1688                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1689                 free(elem);
1690         }
1691 }
1692
1693 /*
1694  * Extern object.
1695  */
1696 static struct extern_type *
1697 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1698 {
1699         struct extern_type *elem;
1700
1701         TAILQ_FOREACH(elem, &p->extern_types, node)
1702                 if (strcmp(elem->name, name) == 0)
1703                         return elem;
1704
1705         return NULL;
1706 }
1707
1708 static struct extern_type_member_func *
1709 extern_type_member_func_find(struct extern_type *type, const char *name)
1710 {
1711         struct extern_type_member_func *elem;
1712
1713         TAILQ_FOREACH(elem, &type->funcs, node)
1714                 if (strcmp(elem->name, name) == 0)
1715                         return elem;
1716
1717         return NULL;
1718 }
1719
1720 static struct extern_obj *
1721 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1722 {
1723         struct extern_obj *elem;
1724
1725         TAILQ_FOREACH(elem, &p->extern_objs, node)
1726                 if (strcmp(elem->name, name) == 0)
1727                         return elem;
1728
1729         return NULL;
1730 }
1731
1732 static struct extern_type_member_func *
1733 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1734                              const char *name,
1735                              struct extern_obj **obj)
1736 {
1737         struct extern_obj *object;
1738         struct extern_type_member_func *func;
1739         char *object_name, *func_name;
1740
1741         if (name[0] != 'e' || name[1] != '.')
1742                 return NULL;
1743
1744         object_name = strdup(&name[2]);
1745         if (!object_name)
1746                 return NULL;
1747
1748         func_name = strchr(object_name, '.');
1749         if (!func_name) {
1750                 free(object_name);
1751                 return NULL;
1752         }
1753
1754         *func_name = 0;
1755         func_name++;
1756
1757         object = extern_obj_find(p, object_name);
1758         if (!object) {
1759                 free(object_name);
1760                 return NULL;
1761         }
1762
1763         func = extern_type_member_func_find(object->type, func_name);
1764         if (!func) {
1765                 free(object_name);
1766                 return NULL;
1767         }
1768
1769         if (obj)
1770                 *obj = object;
1771
1772         free(object_name);
1773         return func;
1774 }
1775
1776 static struct field *
1777 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1778                                const char *name,
1779                                struct extern_obj **object)
1780 {
1781         struct extern_obj *obj;
1782         struct field *f;
1783         char *obj_name, *field_name;
1784
1785         if ((name[0] != 'e') || (name[1] != '.'))
1786                 return NULL;
1787
1788         obj_name = strdup(&name[2]);
1789         if (!obj_name)
1790                 return NULL;
1791
1792         field_name = strchr(obj_name, '.');
1793         if (!field_name) {
1794                 free(obj_name);
1795                 return NULL;
1796         }
1797
1798         *field_name = 0;
1799         field_name++;
1800
1801         obj = extern_obj_find(p, obj_name);
1802         if (!obj) {
1803                 free(obj_name);
1804                 return NULL;
1805         }
1806
1807         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1808         if (!f) {
1809                 free(obj_name);
1810                 return NULL;
1811         }
1812
1813         if (object)
1814                 *object = obj;
1815
1816         free(obj_name);
1817         return f;
1818 }
1819
1820 int
1821 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1822         const char *name,
1823         const char *mailbox_struct_type_name,
1824         rte_swx_extern_type_constructor_t constructor,
1825         rte_swx_extern_type_destructor_t destructor)
1826 {
1827         struct extern_type *elem;
1828         struct struct_type *mailbox_struct_type;
1829
1830         CHECK(p, EINVAL);
1831
1832         CHECK_NAME(name, EINVAL);
1833         CHECK(!extern_type_find(p, name), EEXIST);
1834
1835         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1836         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1837         CHECK(mailbox_struct_type, EINVAL);
1838
1839         CHECK(constructor, EINVAL);
1840         CHECK(destructor, EINVAL);
1841
1842         /* Node allocation. */
1843         elem = calloc(1, sizeof(struct extern_type));
1844         CHECK(elem, ENOMEM);
1845
1846         /* Node initialization. */
1847         strcpy(elem->name, name);
1848         elem->mailbox_struct_type = mailbox_struct_type;
1849         elem->constructor = constructor;
1850         elem->destructor = destructor;
1851         TAILQ_INIT(&elem->funcs);
1852
1853         /* Node add to tailq. */
1854         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1855
1856         return 0;
1857 }
1858
1859 int
1860 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1861         const char *extern_type_name,
1862         const char *name,
1863         rte_swx_extern_type_member_func_t member_func)
1864 {
1865         struct extern_type *type;
1866         struct extern_type_member_func *type_member;
1867
1868         CHECK(p, EINVAL);
1869
1870         CHECK_NAME(extern_type_name, EINVAL);
1871         type = extern_type_find(p, extern_type_name);
1872         CHECK(type, EINVAL);
1873         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1874
1875         CHECK_NAME(name, EINVAL);
1876         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1877
1878         CHECK(member_func, EINVAL);
1879
1880         /* Node allocation. */
1881         type_member = calloc(1, sizeof(struct extern_type_member_func));
1882         CHECK(type_member, ENOMEM);
1883
1884         /* Node initialization. */
1885         strcpy(type_member->name, name);
1886         type_member->func = member_func;
1887         type_member->id = type->n_funcs;
1888
1889         /* Node add to tailq. */
1890         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1891         type->n_funcs++;
1892
1893         return 0;
1894 }
1895
1896 int
1897 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1898                                       const char *extern_type_name,
1899                                       const char *name,
1900                                       const char *args)
1901 {
1902         struct extern_type *type;
1903         struct extern_obj *obj;
1904         void *obj_handle;
1905
1906         CHECK(p, EINVAL);
1907
1908         CHECK_NAME(extern_type_name, EINVAL);
1909         type = extern_type_find(p, extern_type_name);
1910         CHECK(type, EINVAL);
1911
1912         CHECK_NAME(name, EINVAL);
1913         CHECK(!extern_obj_find(p, name), EEXIST);
1914
1915         /* Node allocation. */
1916         obj = calloc(1, sizeof(struct extern_obj));
1917         CHECK(obj, ENOMEM);
1918
1919         /* Object construction. */
1920         obj_handle = type->constructor(args);
1921         if (!obj_handle) {
1922                 free(obj);
1923                 CHECK(0, ENODEV);
1924         }
1925
1926         /* Node initialization. */
1927         strcpy(obj->name, name);
1928         obj->type = type;
1929         obj->obj = obj_handle;
1930         obj->struct_id = p->n_structs;
1931         obj->id = p->n_extern_objs;
1932
1933         /* Node add to tailq. */
1934         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1935         p->n_extern_objs++;
1936         p->n_structs++;
1937
1938         return 0;
1939 }
1940
1941 static int
1942 extern_obj_build(struct rte_swx_pipeline *p)
1943 {
1944         uint32_t i;
1945
1946         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1947                 struct thread *t = &p->threads[i];
1948                 struct extern_obj *obj;
1949
1950                 t->extern_objs = calloc(p->n_extern_objs,
1951                                         sizeof(struct extern_obj_runtime));
1952                 CHECK(t->extern_objs, ENOMEM);
1953
1954                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1955                         struct extern_obj_runtime *r =
1956                                 &t->extern_objs[obj->id];
1957                         struct extern_type_member_func *func;
1958                         uint32_t mailbox_size =
1959                                 obj->type->mailbox_struct_type->n_bits / 8;
1960
1961                         r->obj = obj->obj;
1962
1963                         r->mailbox = calloc(1, mailbox_size);
1964                         CHECK(r->mailbox, ENOMEM);
1965
1966                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1967                                 r->funcs[func->id] = func->func;
1968
1969                         t->structs[obj->struct_id] = r->mailbox;
1970                 }
1971         }
1972
1973         return 0;
1974 }
1975
1976 static void
1977 extern_obj_build_free(struct rte_swx_pipeline *p)
1978 {
1979         uint32_t i;
1980
1981         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1982                 struct thread *t = &p->threads[i];
1983                 uint32_t j;
1984
1985                 if (!t->extern_objs)
1986                         continue;
1987
1988                 for (j = 0; j < p->n_extern_objs; j++) {
1989                         struct extern_obj_runtime *r = &t->extern_objs[j];
1990
1991                         free(r->mailbox);
1992                 }
1993
1994                 free(t->extern_objs);
1995                 t->extern_objs = NULL;
1996         }
1997 }
1998
1999 static void
2000 extern_obj_free(struct rte_swx_pipeline *p)
2001 {
2002         extern_obj_build_free(p);
2003
2004         /* Extern objects. */
2005         for ( ; ; ) {
2006                 struct extern_obj *elem;
2007
2008                 elem = TAILQ_FIRST(&p->extern_objs);
2009                 if (!elem)
2010                         break;
2011
2012                 TAILQ_REMOVE(&p->extern_objs, elem, node);
2013                 if (elem->obj)
2014                         elem->type->destructor(elem->obj);
2015                 free(elem);
2016         }
2017
2018         /* Extern types. */
2019         for ( ; ; ) {
2020                 struct extern_type *elem;
2021
2022                 elem = TAILQ_FIRST(&p->extern_types);
2023                 if (!elem)
2024                         break;
2025
2026                 TAILQ_REMOVE(&p->extern_types, elem, node);
2027
2028                 for ( ; ; ) {
2029                         struct extern_type_member_func *func;
2030
2031                         func = TAILQ_FIRST(&elem->funcs);
2032                         if (!func)
2033                                 break;
2034
2035                         TAILQ_REMOVE(&elem->funcs, func, node);
2036                         free(func);
2037                 }
2038
2039                 free(elem);
2040         }
2041 }
2042
2043 /*
2044  * Extern function.
2045  */
2046 static struct extern_func *
2047 extern_func_find(struct rte_swx_pipeline *p, const char *name)
2048 {
2049         struct extern_func *elem;
2050
2051         TAILQ_FOREACH(elem, &p->extern_funcs, node)
2052                 if (strcmp(elem->name, name) == 0)
2053                         return elem;
2054
2055         return NULL;
2056 }
2057
2058 static struct extern_func *
2059 extern_func_parse(struct rte_swx_pipeline *p,
2060                   const char *name)
2061 {
2062         if (name[0] != 'f' || name[1] != '.')
2063                 return NULL;
2064
2065         return extern_func_find(p, &name[2]);
2066 }
2067
2068 static struct field *
2069 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
2070                                 const char *name,
2071                                 struct extern_func **function)
2072 {
2073         struct extern_func *func;
2074         struct field *f;
2075         char *func_name, *field_name;
2076
2077         if ((name[0] != 'f') || (name[1] != '.'))
2078                 return NULL;
2079
2080         func_name = strdup(&name[2]);
2081         if (!func_name)
2082                 return NULL;
2083
2084         field_name = strchr(func_name, '.');
2085         if (!field_name) {
2086                 free(func_name);
2087                 return NULL;
2088         }
2089
2090         *field_name = 0;
2091         field_name++;
2092
2093         func = extern_func_find(p, func_name);
2094         if (!func) {
2095                 free(func_name);
2096                 return NULL;
2097         }
2098
2099         f = struct_type_field_find(func->mailbox_struct_type, field_name);
2100         if (!f) {
2101                 free(func_name);
2102                 return NULL;
2103         }
2104
2105         if (function)
2106                 *function = func;
2107
2108         free(func_name);
2109         return f;
2110 }
2111
2112 int
2113 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
2114                                       const char *name,
2115                                       const char *mailbox_struct_type_name,
2116                                       rte_swx_extern_func_t func)
2117 {
2118         struct extern_func *f;
2119         struct struct_type *mailbox_struct_type;
2120
2121         CHECK(p, EINVAL);
2122
2123         CHECK_NAME(name, EINVAL);
2124         CHECK(!extern_func_find(p, name), EEXIST);
2125
2126         CHECK_NAME(mailbox_struct_type_name, EINVAL);
2127         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2128         CHECK(mailbox_struct_type, EINVAL);
2129
2130         CHECK(func, EINVAL);
2131
2132         /* Node allocation. */
2133         f = calloc(1, sizeof(struct extern_func));
2134         CHECK(func, ENOMEM);
2135
2136         /* Node initialization. */
2137         strcpy(f->name, name);
2138         f->mailbox_struct_type = mailbox_struct_type;
2139         f->func = func;
2140         f->struct_id = p->n_structs;
2141         f->id = p->n_extern_funcs;
2142
2143         /* Node add to tailq. */
2144         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
2145         p->n_extern_funcs++;
2146         p->n_structs++;
2147
2148         return 0;
2149 }
2150
2151 static int
2152 extern_func_build(struct rte_swx_pipeline *p)
2153 {
2154         uint32_t i;
2155
2156         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2157                 struct thread *t = &p->threads[i];
2158                 struct extern_func *func;
2159
2160                 /* Memory allocation. */
2161                 t->extern_funcs = calloc(p->n_extern_funcs,
2162                                          sizeof(struct extern_func_runtime));
2163                 CHECK(t->extern_funcs, ENOMEM);
2164
2165                 /* Extern function. */
2166                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
2167                         struct extern_func_runtime *r =
2168                                 &t->extern_funcs[func->id];
2169                         uint32_t mailbox_size =
2170                                 func->mailbox_struct_type->n_bits / 8;
2171
2172                         r->func = func->func;
2173
2174                         r->mailbox = calloc(1, mailbox_size);
2175                         CHECK(r->mailbox, ENOMEM);
2176
2177                         t->structs[func->struct_id] = r->mailbox;
2178                 }
2179         }
2180
2181         return 0;
2182 }
2183
2184 static void
2185 extern_func_build_free(struct rte_swx_pipeline *p)
2186 {
2187         uint32_t i;
2188
2189         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2190                 struct thread *t = &p->threads[i];
2191                 uint32_t j;
2192
2193                 if (!t->extern_funcs)
2194                         continue;
2195
2196                 for (j = 0; j < p->n_extern_funcs; j++) {
2197                         struct extern_func_runtime *r = &t->extern_funcs[j];
2198
2199                         free(r->mailbox);
2200                 }
2201
2202                 free(t->extern_funcs);
2203                 t->extern_funcs = NULL;
2204         }
2205 }
2206
2207 static void
2208 extern_func_free(struct rte_swx_pipeline *p)
2209 {
2210         extern_func_build_free(p);
2211
2212         for ( ; ; ) {
2213                 struct extern_func *elem;
2214
2215                 elem = TAILQ_FIRST(&p->extern_funcs);
2216                 if (!elem)
2217                         break;
2218
2219                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
2220                 free(elem);
2221         }
2222 }
2223
2224 /*
2225  * Header.
2226  */
2227 static struct header *
2228 header_find(struct rte_swx_pipeline *p, const char *name)
2229 {
2230         struct header *elem;
2231
2232         TAILQ_FOREACH(elem, &p->headers, node)
2233                 if (strcmp(elem->name, name) == 0)
2234                         return elem;
2235
2236         return NULL;
2237 }
2238
2239 static struct header *
2240 header_parse(struct rte_swx_pipeline *p,
2241              const char *name)
2242 {
2243         if (name[0] != 'h' || name[1] != '.')
2244                 return NULL;
2245
2246         return header_find(p, &name[2]);
2247 }
2248
2249 static struct field *
2250 header_field_parse(struct rte_swx_pipeline *p,
2251                    const char *name,
2252                    struct header **header)
2253 {
2254         struct header *h;
2255         struct field *f;
2256         char *header_name, *field_name;
2257
2258         if ((name[0] != 'h') || (name[1] != '.'))
2259                 return NULL;
2260
2261         header_name = strdup(&name[2]);
2262         if (!header_name)
2263                 return NULL;
2264
2265         field_name = strchr(header_name, '.');
2266         if (!field_name) {
2267                 free(header_name);
2268                 return NULL;
2269         }
2270
2271         *field_name = 0;
2272         field_name++;
2273
2274         h = header_find(p, header_name);
2275         if (!h) {
2276                 free(header_name);
2277                 return NULL;
2278         }
2279
2280         f = struct_type_field_find(h->st, field_name);
2281         if (!f) {
2282                 free(header_name);
2283                 return NULL;
2284         }
2285
2286         if (header)
2287                 *header = h;
2288
2289         free(header_name);
2290         return f;
2291 }
2292
2293 int
2294 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2295                                         const char *name,
2296                                         const char *struct_type_name)
2297 {
2298         struct struct_type *st;
2299         struct header *h;
2300         size_t n_headers_max;
2301
2302         CHECK(p, EINVAL);
2303         CHECK_NAME(name, EINVAL);
2304         CHECK_NAME(struct_type_name, EINVAL);
2305
2306         CHECK(!header_find(p, name), EEXIST);
2307
2308         st = struct_type_find(p, struct_type_name);
2309         CHECK(st, EINVAL);
2310
2311         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2312         CHECK(p->n_headers < n_headers_max, ENOSPC);
2313
2314         /* Node allocation. */
2315         h = calloc(1, sizeof(struct header));
2316         CHECK(h, ENOMEM);
2317
2318         /* Node initialization. */
2319         strcpy(h->name, name);
2320         h->st = st;
2321         h->struct_id = p->n_structs;
2322         h->id = p->n_headers;
2323
2324         /* Node add to tailq. */
2325         TAILQ_INSERT_TAIL(&p->headers, h, node);
2326         p->n_headers++;
2327         p->n_structs++;
2328
2329         return 0;
2330 }
2331
2332 static int
2333 header_build(struct rte_swx_pipeline *p)
2334 {
2335         struct header *h;
2336         uint32_t n_bytes = 0, i;
2337
2338         TAILQ_FOREACH(h, &p->headers, node) {
2339                 n_bytes += h->st->n_bits / 8;
2340         }
2341
2342         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2343                 struct thread *t = &p->threads[i];
2344                 uint32_t offset = 0;
2345
2346                 t->headers = calloc(p->n_headers,
2347                                     sizeof(struct header_runtime));
2348                 CHECK(t->headers, ENOMEM);
2349
2350                 t->headers_out = calloc(p->n_headers,
2351                                         sizeof(struct header_out_runtime));
2352                 CHECK(t->headers_out, ENOMEM);
2353
2354                 t->header_storage = calloc(1, n_bytes);
2355                 CHECK(t->header_storage, ENOMEM);
2356
2357                 t->header_out_storage = calloc(1, n_bytes);
2358                 CHECK(t->header_out_storage, ENOMEM);
2359
2360                 TAILQ_FOREACH(h, &p->headers, node) {
2361                         uint8_t *header_storage;
2362
2363                         header_storage = &t->header_storage[offset];
2364                         offset += h->st->n_bits / 8;
2365
2366                         t->headers[h->id].ptr0 = header_storage;
2367                         t->structs[h->struct_id] = header_storage;
2368                 }
2369         }
2370
2371         return 0;
2372 }
2373
2374 static void
2375 header_build_free(struct rte_swx_pipeline *p)
2376 {
2377         uint32_t i;
2378
2379         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2380                 struct thread *t = &p->threads[i];
2381
2382                 free(t->headers_out);
2383                 t->headers_out = NULL;
2384
2385                 free(t->headers);
2386                 t->headers = NULL;
2387
2388                 free(t->header_out_storage);
2389                 t->header_out_storage = NULL;
2390
2391                 free(t->header_storage);
2392                 t->header_storage = NULL;
2393         }
2394 }
2395
2396 static void
2397 header_free(struct rte_swx_pipeline *p)
2398 {
2399         header_build_free(p);
2400
2401         for ( ; ; ) {
2402                 struct header *elem;
2403
2404                 elem = TAILQ_FIRST(&p->headers);
2405                 if (!elem)
2406                         break;
2407
2408                 TAILQ_REMOVE(&p->headers, elem, node);
2409                 free(elem);
2410         }
2411 }
2412
2413 /*
2414  * Meta-data.
2415  */
2416 static struct field *
2417 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2418 {
2419         if (!p->metadata_st)
2420                 return NULL;
2421
2422         if (name[0] != 'm' || name[1] != '.')
2423                 return NULL;
2424
2425         return struct_type_field_find(p->metadata_st, &name[2]);
2426 }
2427
2428 int
2429 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2430                                           const char *struct_type_name)
2431 {
2432         struct struct_type *st = NULL;
2433
2434         CHECK(p, EINVAL);
2435
2436         CHECK_NAME(struct_type_name, EINVAL);
2437         st  = struct_type_find(p, struct_type_name);
2438         CHECK(st, EINVAL);
2439         CHECK(!p->metadata_st, EINVAL);
2440
2441         p->metadata_st = st;
2442         p->metadata_struct_id = p->n_structs;
2443
2444         p->n_structs++;
2445
2446         return 0;
2447 }
2448
2449 static int
2450 metadata_build(struct rte_swx_pipeline *p)
2451 {
2452         uint32_t n_bytes = p->metadata_st->n_bits / 8;
2453         uint32_t i;
2454
2455         /* Thread-level initialization. */
2456         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2457                 struct thread *t = &p->threads[i];
2458                 uint8_t *metadata;
2459
2460                 metadata = calloc(1, n_bytes);
2461                 CHECK(metadata, ENOMEM);
2462
2463                 t->metadata = metadata;
2464                 t->structs[p->metadata_struct_id] = metadata;
2465         }
2466
2467         return 0;
2468 }
2469
2470 static void
2471 metadata_build_free(struct rte_swx_pipeline *p)
2472 {
2473         uint32_t i;
2474
2475         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2476                 struct thread *t = &p->threads[i];
2477
2478                 free(t->metadata);
2479                 t->metadata = NULL;
2480         }
2481 }
2482
2483 static void
2484 metadata_free(struct rte_swx_pipeline *p)
2485 {
2486         metadata_build_free(p);
2487 }
2488
2489 /*
2490  * Instruction.
2491  */
2492 static int
2493 instruction_is_jmp(struct instruction *instr)
2494 {
2495         switch (instr->type) {
2496         case INSTR_JMP:
2497         case INSTR_JMP_VALID:
2498         case INSTR_JMP_INVALID:
2499         case INSTR_JMP_HIT:
2500         case INSTR_JMP_MISS:
2501         case INSTR_JMP_ACTION_HIT:
2502         case INSTR_JMP_ACTION_MISS:
2503         case INSTR_JMP_EQ:
2504         case INSTR_JMP_EQ_S:
2505         case INSTR_JMP_EQ_I:
2506         case INSTR_JMP_NEQ:
2507         case INSTR_JMP_NEQ_S:
2508         case INSTR_JMP_NEQ_I:
2509         case INSTR_JMP_LT:
2510         case INSTR_JMP_LT_MH:
2511         case INSTR_JMP_LT_HM:
2512         case INSTR_JMP_LT_HH:
2513         case INSTR_JMP_LT_MI:
2514         case INSTR_JMP_LT_HI:
2515         case INSTR_JMP_GT:
2516         case INSTR_JMP_GT_MH:
2517         case INSTR_JMP_GT_HM:
2518         case INSTR_JMP_GT_HH:
2519         case INSTR_JMP_GT_MI:
2520         case INSTR_JMP_GT_HI:
2521                 return 1;
2522
2523         default:
2524                 return 0;
2525         }
2526 }
2527
2528 static struct field *
2529 action_field_parse(struct action *action, const char *name);
2530
2531 static struct field *
2532 struct_field_parse(struct rte_swx_pipeline *p,
2533                    struct action *action,
2534                    const char *name,
2535                    uint32_t *struct_id)
2536 {
2537         struct field *f;
2538
2539         switch (name[0]) {
2540         case 'h':
2541         {
2542                 struct header *header;
2543
2544                 f = header_field_parse(p, name, &header);
2545                 if (!f)
2546                         return NULL;
2547
2548                 *struct_id = header->struct_id;
2549                 return f;
2550         }
2551
2552         case 'm':
2553         {
2554                 f = metadata_field_parse(p, name);
2555                 if (!f)
2556                         return NULL;
2557
2558                 *struct_id = p->metadata_struct_id;
2559                 return f;
2560         }
2561
2562         case 't':
2563         {
2564                 if (!action)
2565                         return NULL;
2566
2567                 f = action_field_parse(action, name);
2568                 if (!f)
2569                         return NULL;
2570
2571                 *struct_id = 0;
2572                 return f;
2573         }
2574
2575         case 'e':
2576         {
2577                 struct extern_obj *obj;
2578
2579                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2580                 if (!f)
2581                         return NULL;
2582
2583                 *struct_id = obj->struct_id;
2584                 return f;
2585         }
2586
2587         case 'f':
2588         {
2589                 struct extern_func *func;
2590
2591                 f = extern_func_mailbox_field_parse(p, name, &func);
2592                 if (!f)
2593                         return NULL;
2594
2595                 *struct_id = func->struct_id;
2596                 return f;
2597         }
2598
2599         default:
2600                 return NULL;
2601         }
2602 }
2603
2604 static inline void
2605 pipeline_port_inc(struct rte_swx_pipeline *p)
2606 {
2607         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2608 }
2609
2610 static inline void
2611 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2612 {
2613         t->ip = p->instructions;
2614 }
2615
2616 static inline void
2617 thread_ip_set(struct thread *t, struct instruction *ip)
2618 {
2619         t->ip = ip;
2620 }
2621
2622 static inline void
2623 thread_ip_action_call(struct rte_swx_pipeline *p,
2624                       struct thread *t,
2625                       uint32_t action_id)
2626 {
2627         t->ret = t->ip + 1;
2628         t->ip = p->action_instructions[action_id];
2629 }
2630
2631 static inline void
2632 thread_ip_inc(struct rte_swx_pipeline *p);
2633
2634 static inline void
2635 thread_ip_inc(struct rte_swx_pipeline *p)
2636 {
2637         struct thread *t = &p->threads[p->thread_id];
2638
2639         t->ip++;
2640 }
2641
2642 static inline void
2643 thread_ip_inc_cond(struct thread *t, int cond)
2644 {
2645         t->ip += cond;
2646 }
2647
2648 static inline void
2649 thread_yield(struct rte_swx_pipeline *p)
2650 {
2651         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2652 }
2653
2654 static inline void
2655 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2656 {
2657         p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2658 }
2659
2660 /*
2661  * rx.
2662  */
2663 static int
2664 instr_rx_translate(struct rte_swx_pipeline *p,
2665                    struct action *action,
2666                    char **tokens,
2667                    int n_tokens,
2668                    struct instruction *instr,
2669                    struct instruction_data *data __rte_unused)
2670 {
2671         struct field *f;
2672
2673         CHECK(!action, EINVAL);
2674         CHECK(n_tokens == 2, EINVAL);
2675
2676         f = metadata_field_parse(p, tokens[1]);
2677         CHECK(f, EINVAL);
2678
2679         instr->type = INSTR_RX;
2680         instr->io.io.offset = f->offset / 8;
2681         instr->io.io.n_bits = f->n_bits;
2682         return 0;
2683 }
2684
2685 static inline void
2686 instr_rx_exec(struct rte_swx_pipeline *p);
2687
2688 static inline void
2689 instr_rx_exec(struct rte_swx_pipeline *p)
2690 {
2691         struct thread *t = &p->threads[p->thread_id];
2692         struct instruction *ip = t->ip;
2693         struct port_in_runtime *port = &p->in[p->port_id];
2694         struct rte_swx_pkt *pkt = &t->pkt;
2695         int pkt_received;
2696
2697         /* Packet. */
2698         pkt_received = port->pkt_rx(port->obj, pkt);
2699         t->ptr = &pkt->pkt[pkt->offset];
2700         rte_prefetch0(t->ptr);
2701
2702         TRACE("[Thread %2u] rx %s from port %u\n",
2703               p->thread_id,
2704               pkt_received ? "1 pkt" : "0 pkts",
2705               p->port_id);
2706
2707         /* Headers. */
2708         t->valid_headers = 0;
2709         t->n_headers_out = 0;
2710
2711         /* Meta-data. */
2712         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2713
2714         /* Tables. */
2715         t->table_state = p->table_state;
2716
2717         /* Thread. */
2718         pipeline_port_inc(p);
2719         thread_ip_inc_cond(t, pkt_received);
2720         thread_yield(p);
2721 }
2722
2723 /*
2724  * tx.
2725  */
2726 static int
2727 instr_tx_translate(struct rte_swx_pipeline *p,
2728                    struct action *action __rte_unused,
2729                    char **tokens,
2730                    int n_tokens,
2731                    struct instruction *instr,
2732                    struct instruction_data *data __rte_unused)
2733 {
2734         struct field *f;
2735
2736         CHECK(n_tokens == 2, EINVAL);
2737
2738         f = metadata_field_parse(p, tokens[1]);
2739         CHECK(f, EINVAL);
2740
2741         instr->type = INSTR_TX;
2742         instr->io.io.offset = f->offset / 8;
2743         instr->io.io.n_bits = f->n_bits;
2744         return 0;
2745 }
2746
2747 static inline void
2748 emit_handler(struct thread *t)
2749 {
2750         struct header_out_runtime *h0 = &t->headers_out[0];
2751         struct header_out_runtime *h1 = &t->headers_out[1];
2752         uint32_t offset = 0, i;
2753
2754         /* No header change or header decapsulation. */
2755         if ((t->n_headers_out == 1) &&
2756             (h0->ptr + h0->n_bytes == t->ptr)) {
2757                 TRACE("Emit handler: no header change or header decap.\n");
2758
2759                 t->pkt.offset -= h0->n_bytes;
2760                 t->pkt.length += h0->n_bytes;
2761
2762                 return;
2763         }
2764
2765         /* Header encapsulation (optionally, with prior header decasulation). */
2766         if ((t->n_headers_out == 2) &&
2767             (h1->ptr + h1->n_bytes == t->ptr) &&
2768             (h0->ptr == h0->ptr0)) {
2769                 uint32_t offset;
2770
2771                 TRACE("Emit handler: header encapsulation.\n");
2772
2773                 offset = h0->n_bytes + h1->n_bytes;
2774                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2775                 t->pkt.offset -= offset;
2776                 t->pkt.length += offset;
2777
2778                 return;
2779         }
2780
2781         /* Header insertion. */
2782         /* TBD */
2783
2784         /* Header extraction. */
2785         /* TBD */
2786
2787         /* For any other case. */
2788         TRACE("Emit handler: complex case.\n");
2789
2790         for (i = 0; i < t->n_headers_out; i++) {
2791                 struct header_out_runtime *h = &t->headers_out[i];
2792
2793                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2794                 offset += h->n_bytes;
2795         }
2796
2797         if (offset) {
2798                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2799                 t->pkt.offset -= offset;
2800                 t->pkt.length += offset;
2801         }
2802 }
2803
2804 static inline void
2805 instr_tx_exec(struct rte_swx_pipeline *p);
2806
2807 static inline void
2808 instr_tx_exec(struct rte_swx_pipeline *p)
2809 {
2810         struct thread *t = &p->threads[p->thread_id];
2811         struct instruction *ip = t->ip;
2812         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2813         struct port_out_runtime *port = &p->out[port_id];
2814         struct rte_swx_pkt *pkt = &t->pkt;
2815
2816         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2817               p->thread_id,
2818               (uint32_t)port_id);
2819
2820         /* Headers. */
2821         emit_handler(t);
2822
2823         /* Packet. */
2824         port->pkt_tx(port->obj, pkt);
2825
2826         /* Thread. */
2827         thread_ip_reset(p, t);
2828         instr_rx_exec(p);
2829 }
2830
2831 /*
2832  * extract.
2833  */
2834 static int
2835 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2836                             struct action *action,
2837                             char **tokens,
2838                             int n_tokens,
2839                             struct instruction *instr,
2840                             struct instruction_data *data __rte_unused)
2841 {
2842         struct header *h;
2843
2844         CHECK(!action, EINVAL);
2845         CHECK(n_tokens == 2, EINVAL);
2846
2847         h = header_parse(p, tokens[1]);
2848         CHECK(h, EINVAL);
2849
2850         instr->type = INSTR_HDR_EXTRACT;
2851         instr->io.hdr.header_id[0] = h->id;
2852         instr->io.hdr.struct_id[0] = h->struct_id;
2853         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2854         return 0;
2855 }
2856
2857 static inline void
2858 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2859
2860 static inline void
2861 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2862 {
2863         struct thread *t = &p->threads[p->thread_id];
2864         struct instruction *ip = t->ip;
2865         uint64_t valid_headers = t->valid_headers;
2866         uint8_t *ptr = t->ptr;
2867         uint32_t offset = t->pkt.offset;
2868         uint32_t length = t->pkt.length;
2869         uint32_t i;
2870
2871         for (i = 0; i < n_extract; i++) {
2872                 uint32_t header_id = ip->io.hdr.header_id[i];
2873                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2874                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2875
2876                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2877                       p->thread_id,
2878                       header_id,
2879                       n_bytes);
2880
2881                 /* Headers. */
2882                 t->structs[struct_id] = ptr;
2883                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2884
2885                 /* Packet. */
2886                 offset += n_bytes;
2887                 length -= n_bytes;
2888                 ptr += n_bytes;
2889         }
2890
2891         /* Headers. */
2892         t->valid_headers = valid_headers;
2893
2894         /* Packet. */
2895         t->pkt.offset = offset;
2896         t->pkt.length = length;
2897         t->ptr = ptr;
2898 }
2899
2900 static inline void
2901 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2902 {
2903         __instr_hdr_extract_exec(p, 1);
2904
2905         /* Thread. */
2906         thread_ip_inc(p);
2907 }
2908
2909 static inline void
2910 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2911 {
2912         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2913               p->thread_id);
2914
2915         __instr_hdr_extract_exec(p, 2);
2916
2917         /* Thread. */
2918         thread_ip_inc(p);
2919 }
2920
2921 static inline void
2922 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2923 {
2924         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2925               p->thread_id);
2926
2927         __instr_hdr_extract_exec(p, 3);
2928
2929         /* Thread. */
2930         thread_ip_inc(p);
2931 }
2932
2933 static inline void
2934 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2935 {
2936         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2937               p->thread_id);
2938
2939         __instr_hdr_extract_exec(p, 4);
2940
2941         /* Thread. */
2942         thread_ip_inc(p);
2943 }
2944
2945 static inline void
2946 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2947 {
2948         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2949               p->thread_id);
2950
2951         __instr_hdr_extract_exec(p, 5);
2952
2953         /* Thread. */
2954         thread_ip_inc(p);
2955 }
2956
2957 static inline void
2958 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2959 {
2960         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2961               p->thread_id);
2962
2963         __instr_hdr_extract_exec(p, 6);
2964
2965         /* Thread. */
2966         thread_ip_inc(p);
2967 }
2968
2969 static inline void
2970 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2971 {
2972         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2973               p->thread_id);
2974
2975         __instr_hdr_extract_exec(p, 7);
2976
2977         /* Thread. */
2978         thread_ip_inc(p);
2979 }
2980
2981 static inline void
2982 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2983 {
2984         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2985               p->thread_id);
2986
2987         __instr_hdr_extract_exec(p, 8);
2988
2989         /* Thread. */
2990         thread_ip_inc(p);
2991 }
2992
2993 /*
2994  * emit.
2995  */
2996 static int
2997 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2998                          struct action *action __rte_unused,
2999                          char **tokens,
3000                          int n_tokens,
3001                          struct instruction *instr,
3002                          struct instruction_data *data __rte_unused)
3003 {
3004         struct header *h;
3005
3006         CHECK(n_tokens == 2, EINVAL);
3007
3008         h = header_parse(p, tokens[1]);
3009         CHECK(h, EINVAL);
3010
3011         instr->type = INSTR_HDR_EMIT;
3012         instr->io.hdr.header_id[0] = h->id;
3013         instr->io.hdr.struct_id[0] = h->struct_id;
3014         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3015         return 0;
3016 }
3017
3018 static inline void
3019 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
3020
3021 static inline void
3022 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
3023 {
3024         struct thread *t = &p->threads[p->thread_id];
3025         struct instruction *ip = t->ip;
3026         uint32_t n_headers_out = t->n_headers_out;
3027         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
3028         uint8_t *ho_ptr = NULL;
3029         uint32_t ho_nbytes = 0, i;
3030
3031         for (i = 0; i < n_emit; i++) {
3032                 uint32_t header_id = ip->io.hdr.header_id[i];
3033                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3034                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3035
3036                 struct header_runtime *hi = &t->headers[header_id];
3037                 uint8_t *hi_ptr = t->structs[struct_id];
3038
3039                 TRACE("[Thread %2u]: emit header %u\n",
3040                       p->thread_id,
3041                       header_id);
3042
3043                 /* Headers. */
3044                 if (!i) {
3045                         if (!t->n_headers_out) {
3046                                 ho = &t->headers_out[0];
3047
3048                                 ho->ptr0 = hi->ptr0;
3049                                 ho->ptr = hi_ptr;
3050
3051                                 ho_ptr = hi_ptr;
3052                                 ho_nbytes = n_bytes;
3053
3054                                 n_headers_out = 1;
3055
3056                                 continue;
3057                         } else {
3058                                 ho_ptr = ho->ptr;
3059                                 ho_nbytes = ho->n_bytes;
3060                         }
3061                 }
3062
3063                 if (ho_ptr + ho_nbytes == hi_ptr) {
3064                         ho_nbytes += n_bytes;
3065                 } else {
3066                         ho->n_bytes = ho_nbytes;
3067
3068                         ho++;
3069                         ho->ptr0 = hi->ptr0;
3070                         ho->ptr = hi_ptr;
3071
3072                         ho_ptr = hi_ptr;
3073                         ho_nbytes = n_bytes;
3074
3075                         n_headers_out++;
3076                 }
3077         }
3078
3079         ho->n_bytes = ho_nbytes;
3080         t->n_headers_out = n_headers_out;
3081 }
3082
3083 static inline void
3084 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
3085 {
3086         __instr_hdr_emit_exec(p, 1);
3087
3088         /* Thread. */
3089         thread_ip_inc(p);
3090 }
3091
3092 static inline void
3093 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
3094 {
3095         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3096               p->thread_id);
3097
3098         __instr_hdr_emit_exec(p, 1);
3099         instr_tx_exec(p);
3100 }
3101
3102 static inline void
3103 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
3104 {
3105         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3106               p->thread_id);
3107
3108         __instr_hdr_emit_exec(p, 2);
3109         instr_tx_exec(p);
3110 }
3111
3112 static inline void
3113 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
3114 {
3115         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3116               p->thread_id);
3117
3118         __instr_hdr_emit_exec(p, 3);
3119         instr_tx_exec(p);
3120 }
3121
3122 static inline void
3123 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
3124 {
3125         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3126               p->thread_id);
3127
3128         __instr_hdr_emit_exec(p, 4);
3129         instr_tx_exec(p);
3130 }
3131
3132 static inline void
3133 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
3134 {
3135         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3136               p->thread_id);
3137
3138         __instr_hdr_emit_exec(p, 5);
3139         instr_tx_exec(p);
3140 }
3141
3142 static inline void
3143 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
3144 {
3145         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3146               p->thread_id);
3147
3148         __instr_hdr_emit_exec(p, 6);
3149         instr_tx_exec(p);
3150 }
3151
3152 static inline void
3153 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
3154 {
3155         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3156               p->thread_id);
3157
3158         __instr_hdr_emit_exec(p, 7);
3159         instr_tx_exec(p);
3160 }
3161
3162 static inline void
3163 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
3164 {
3165         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
3166               p->thread_id);
3167
3168         __instr_hdr_emit_exec(p, 8);
3169         instr_tx_exec(p);
3170 }
3171
3172 /*
3173  * validate.
3174  */
3175 static int
3176 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
3177                              struct action *action __rte_unused,
3178                              char **tokens,
3179                              int n_tokens,
3180                              struct instruction *instr,
3181                              struct instruction_data *data __rte_unused)
3182 {
3183         struct header *h;
3184
3185         CHECK(n_tokens == 2, EINVAL);
3186
3187         h = header_parse(p, tokens[1]);
3188         CHECK(h, EINVAL);
3189
3190         instr->type = INSTR_HDR_VALIDATE;
3191         instr->valid.header_id = h->id;
3192         return 0;
3193 }
3194
3195 static inline void
3196 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
3197 {
3198         struct thread *t = &p->threads[p->thread_id];
3199         struct instruction *ip = t->ip;
3200         uint32_t header_id = ip->valid.header_id;
3201
3202         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
3203
3204         /* Headers. */
3205         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
3206
3207         /* Thread. */
3208         thread_ip_inc(p);
3209 }
3210
3211 /*
3212  * invalidate.
3213  */
3214 static int
3215 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
3216                                struct action *action __rte_unused,
3217                                char **tokens,
3218                                int n_tokens,
3219                                struct instruction *instr,
3220                                struct instruction_data *data __rte_unused)
3221 {
3222         struct header *h;
3223
3224         CHECK(n_tokens == 2, EINVAL);
3225
3226         h = header_parse(p, tokens[1]);
3227         CHECK(h, EINVAL);
3228
3229         instr->type = INSTR_HDR_INVALIDATE;
3230         instr->valid.header_id = h->id;
3231         return 0;
3232 }
3233
3234 static inline void
3235 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3236 {
3237         struct thread *t = &p->threads[p->thread_id];
3238         struct instruction *ip = t->ip;
3239         uint32_t header_id = ip->valid.header_id;
3240
3241         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3242
3243         /* Headers. */
3244         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3245
3246         /* Thread. */
3247         thread_ip_inc(p);
3248 }
3249
3250 /*
3251  * table.
3252  */
3253 static struct table *
3254 table_find(struct rte_swx_pipeline *p, const char *name);
3255
3256 static int
3257 instr_table_translate(struct rte_swx_pipeline *p,
3258                       struct action *action,
3259                       char **tokens,
3260                       int n_tokens,
3261                       struct instruction *instr,
3262                       struct instruction_data *data __rte_unused)
3263 {
3264         struct table *t;
3265
3266         CHECK(!action, EINVAL);
3267         CHECK(n_tokens == 2, EINVAL);
3268
3269         t = table_find(p, tokens[1]);
3270         CHECK(t, EINVAL);
3271
3272         instr->type = INSTR_TABLE;
3273         instr->table.table_id = t->id;
3274         return 0;
3275 }
3276
3277 static inline void
3278 instr_table_exec(struct rte_swx_pipeline *p)
3279 {
3280         struct thread *t = &p->threads[p->thread_id];
3281         struct instruction *ip = t->ip;
3282         uint32_t table_id = ip->table.table_id;
3283         struct rte_swx_table_state *ts = &t->table_state[table_id];
3284         struct table_runtime *table = &t->tables[table_id];
3285         uint64_t action_id;
3286         uint8_t *action_data;
3287         int done, hit;
3288
3289         /* Table. */
3290         done = table->func(ts->obj,
3291                            table->mailbox,
3292                            table->key,
3293                            &action_id,
3294                            &action_data,
3295                            &hit);
3296         if (!done) {
3297                 /* Thread. */
3298                 TRACE("[Thread %2u] table %u (not finalized)\n",
3299                       p->thread_id,
3300                       table_id);
3301
3302                 thread_yield(p);
3303                 return;
3304         }
3305
3306         action_id = hit ? action_id : ts->default_action_id;
3307         action_data = hit ? action_data : ts->default_action_data;
3308
3309         TRACE("[Thread %2u] table %u (%s, action %u)\n",
3310               p->thread_id,
3311               table_id,
3312               hit ? "hit" : "miss",
3313               (uint32_t)action_id);
3314
3315         t->action_id = action_id;
3316         t->structs[0] = action_data;
3317         t->hit = hit;
3318
3319         /* Thread. */
3320         thread_ip_action_call(p, t, action_id);
3321 }
3322
3323 /*
3324  * extern.
3325  */
3326 static int
3327 instr_extern_translate(struct rte_swx_pipeline *p,
3328                        struct action *action __rte_unused,
3329                        char **tokens,
3330                        int n_tokens,
3331                        struct instruction *instr,
3332                        struct instruction_data *data __rte_unused)
3333 {
3334         char *token = tokens[1];
3335
3336         CHECK(n_tokens == 2, EINVAL);
3337
3338         if (token[0] == 'e') {
3339                 struct extern_obj *obj;
3340                 struct extern_type_member_func *func;
3341
3342                 func = extern_obj_member_func_parse(p, token, &obj);
3343                 CHECK(func, EINVAL);
3344
3345                 instr->type = INSTR_EXTERN_OBJ;
3346                 instr->ext_obj.ext_obj_id = obj->id;
3347                 instr->ext_obj.func_id = func->id;
3348
3349                 return 0;
3350         }
3351
3352         if (token[0] == 'f') {
3353                 struct extern_func *func;
3354
3355                 func = extern_func_parse(p, token);
3356                 CHECK(func, EINVAL);
3357
3358                 instr->type = INSTR_EXTERN_FUNC;
3359                 instr->ext_func.ext_func_id = func->id;
3360
3361                 return 0;
3362         }
3363
3364         CHECK(0, EINVAL);
3365 }
3366
3367 static inline void
3368 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3369 {
3370         struct thread *t = &p->threads[p->thread_id];
3371         struct instruction *ip = t->ip;
3372         uint32_t obj_id = ip->ext_obj.ext_obj_id;
3373         uint32_t func_id = ip->ext_obj.func_id;
3374         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3375         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3376
3377         TRACE("[Thread %2u] extern obj %u member func %u\n",
3378               p->thread_id,
3379               obj_id,
3380               func_id);
3381
3382         /* Extern object member function execute. */
3383         uint32_t done = func(obj->obj, obj->mailbox);
3384
3385         /* Thread. */
3386         thread_ip_inc_cond(t, done);
3387         thread_yield_cond(p, done ^ 1);
3388 }
3389
3390 static inline void
3391 instr_extern_func_exec(struct rte_swx_pipeline *p)
3392 {
3393         struct thread *t = &p->threads[p->thread_id];
3394         struct instruction *ip = t->ip;
3395         uint32_t ext_func_id = ip->ext_func.ext_func_id;
3396         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3397         rte_swx_extern_func_t func = ext_func->func;
3398
3399         TRACE("[Thread %2u] extern func %u\n",
3400               p->thread_id,
3401               ext_func_id);
3402
3403         /* Extern function execute. */
3404         uint32_t done = func(ext_func->mailbox);
3405
3406         /* Thread. */
3407         thread_ip_inc_cond(t, done);
3408         thread_yield_cond(p, done ^ 1);
3409 }
3410
3411 /*
3412  * mov.
3413  */
3414 static int
3415 instr_mov_translate(struct rte_swx_pipeline *p,
3416                     struct action *action,
3417                     char **tokens,
3418                     int n_tokens,
3419                     struct instruction *instr,
3420                     struct instruction_data *data __rte_unused)
3421 {
3422         char *dst = tokens[1], *src = tokens[2];
3423         struct field *fdst, *fsrc;
3424         uint64_t src_val;
3425         uint32_t dst_struct_id, src_struct_id;
3426
3427         CHECK(n_tokens == 3, EINVAL);
3428
3429         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3430         CHECK(fdst, EINVAL);
3431
3432         /* MOV or MOV_S. */
3433         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3434         if (fsrc) {
3435                 instr->type = INSTR_MOV;
3436                 if ((dst[0] == 'h' && src[0] != 'h') ||
3437                     (dst[0] != 'h' && src[0] == 'h'))
3438                         instr->type = INSTR_MOV_S;
3439
3440                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3441                 instr->mov.dst.n_bits = fdst->n_bits;
3442                 instr->mov.dst.offset = fdst->offset / 8;
3443                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3444                 instr->mov.src.n_bits = fsrc->n_bits;
3445                 instr->mov.src.offset = fsrc->offset / 8;
3446                 return 0;
3447         }
3448
3449         /* MOV_I. */
3450         src_val = strtoull(src, &src, 0);
3451         CHECK(!src[0], EINVAL);
3452
3453         if (dst[0] == 'h')
3454                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3455
3456         instr->type = INSTR_MOV_I;
3457         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3458         instr->mov.dst.n_bits = fdst->n_bits;
3459         instr->mov.dst.offset = fdst->offset / 8;
3460         instr->mov.src_val = src_val;
3461         return 0;
3462 }
3463
3464 static inline void
3465 instr_mov_exec(struct rte_swx_pipeline *p)
3466 {
3467         struct thread *t = &p->threads[p->thread_id];
3468         struct instruction *ip = t->ip;
3469
3470         TRACE("[Thread %2u] mov\n",
3471               p->thread_id);
3472
3473         MOV(t, ip);
3474
3475         /* Thread. */
3476         thread_ip_inc(p);
3477 }
3478
3479 static inline void
3480 instr_mov_s_exec(struct rte_swx_pipeline *p)
3481 {
3482         struct thread *t = &p->threads[p->thread_id];
3483         struct instruction *ip = t->ip;
3484
3485         TRACE("[Thread %2u] mov (s)\n",
3486               p->thread_id);
3487
3488         MOV_S(t, ip);
3489
3490         /* Thread. */
3491         thread_ip_inc(p);
3492 }
3493
3494 static inline void
3495 instr_mov_i_exec(struct rte_swx_pipeline *p)
3496 {
3497         struct thread *t = &p->threads[p->thread_id];
3498         struct instruction *ip = t->ip;
3499
3500         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
3501               p->thread_id,
3502               ip->mov.src_val);
3503
3504         MOV_I(t, ip);
3505
3506         /* Thread. */
3507         thread_ip_inc(p);
3508 }
3509
3510 /*
3511  * dma.
3512  */
3513 static int
3514 instr_dma_translate(struct rte_swx_pipeline *p,
3515                     struct action *action,
3516                     char **tokens,
3517                     int n_tokens,
3518                     struct instruction *instr,
3519                     struct instruction_data *data __rte_unused)
3520 {
3521         char *dst = tokens[1];
3522         char *src = tokens[2];
3523         struct header *h;
3524         struct field *tf;
3525
3526         CHECK(action, EINVAL);
3527         CHECK(n_tokens == 3, EINVAL);
3528
3529         h = header_parse(p, dst);
3530         CHECK(h, EINVAL);
3531
3532         tf = action_field_parse(action, src);
3533         CHECK(tf, EINVAL);
3534
3535         instr->type = INSTR_DMA_HT;
3536         instr->dma.dst.header_id[0] = h->id;
3537         instr->dma.dst.struct_id[0] = h->struct_id;
3538         instr->dma.n_bytes[0] = h->st->n_bits / 8;
3539         instr->dma.src.offset[0] = tf->offset / 8;
3540
3541         return 0;
3542 }
3543
3544 static inline void
3545 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3546
3547 static inline void
3548 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3549 {
3550         struct thread *t = &p->threads[p->thread_id];
3551         struct instruction *ip = t->ip;
3552         uint8_t *action_data = t->structs[0];
3553         uint64_t valid_headers = t->valid_headers;
3554         uint32_t i;
3555
3556         for (i = 0; i < n_dma; i++) {
3557                 uint32_t header_id = ip->dma.dst.header_id[i];
3558                 uint32_t struct_id = ip->dma.dst.struct_id[i];
3559                 uint32_t offset = ip->dma.src.offset[i];
3560                 uint32_t n_bytes = ip->dma.n_bytes[i];
3561
3562                 struct header_runtime *h = &t->headers[header_id];
3563                 uint8_t *h_ptr0 = h->ptr0;
3564                 uint8_t *h_ptr = t->structs[struct_id];
3565
3566                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3567                         h_ptr : h_ptr0;
3568                 void *src = &action_data[offset];
3569
3570                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3571
3572                 /* Headers. */
3573                 memcpy(dst, src, n_bytes);
3574                 t->structs[struct_id] = dst;
3575                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3576         }
3577
3578         t->valid_headers = valid_headers;
3579 }
3580
3581 static inline void
3582 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3583 {
3584         __instr_dma_ht_exec(p, 1);
3585
3586         /* Thread. */
3587         thread_ip_inc(p);
3588 }
3589
3590 static inline void
3591 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3592 {
3593         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3594               p->thread_id);
3595
3596         __instr_dma_ht_exec(p, 2);
3597
3598         /* Thread. */
3599         thread_ip_inc(p);
3600 }
3601
3602 static inline void
3603 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3604 {
3605         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3606               p->thread_id);
3607
3608         __instr_dma_ht_exec(p, 3);
3609
3610         /* Thread. */
3611         thread_ip_inc(p);
3612 }
3613
3614 static inline void
3615 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3616 {
3617         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3618               p->thread_id);
3619
3620         __instr_dma_ht_exec(p, 4);
3621
3622         /* Thread. */
3623         thread_ip_inc(p);
3624 }
3625
3626 static inline void
3627 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3628 {
3629         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3630               p->thread_id);
3631
3632         __instr_dma_ht_exec(p, 5);
3633
3634         /* Thread. */
3635         thread_ip_inc(p);
3636 }
3637
3638 static inline void
3639 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3640 {
3641         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3642               p->thread_id);
3643
3644         __instr_dma_ht_exec(p, 6);
3645
3646         /* Thread. */
3647         thread_ip_inc(p);
3648 }
3649
3650 static inline void
3651 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3652 {
3653         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3654               p->thread_id);
3655
3656         __instr_dma_ht_exec(p, 7);
3657
3658         /* Thread. */
3659         thread_ip_inc(p);
3660 }
3661
3662 static inline void
3663 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3664 {
3665         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3666               p->thread_id);
3667
3668         __instr_dma_ht_exec(p, 8);
3669
3670         /* Thread. */
3671         thread_ip_inc(p);
3672 }
3673
3674 /*
3675  * alu.
3676  */
3677 static int
3678 instr_alu_add_translate(struct rte_swx_pipeline *p,
3679                         struct action *action,
3680                         char **tokens,
3681                         int n_tokens,
3682                         struct instruction *instr,
3683                         struct instruction_data *data __rte_unused)
3684 {
3685         char *dst = tokens[1], *src = tokens[2];
3686         struct field *fdst, *fsrc;
3687         uint64_t src_val;
3688         uint32_t dst_struct_id, src_struct_id;
3689
3690         CHECK(n_tokens == 3, EINVAL);
3691
3692         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3693         CHECK(fdst, EINVAL);
3694
3695         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3696         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3697         if (fsrc) {
3698                 instr->type = INSTR_ALU_ADD;
3699                 if (dst[0] == 'h' && src[0] != 'h')
3700                         instr->type = INSTR_ALU_ADD_HM;
3701                 if (dst[0] != 'h' && src[0] == 'h')
3702                         instr->type = INSTR_ALU_ADD_MH;
3703                 if (dst[0] == 'h' && src[0] == 'h')
3704                         instr->type = INSTR_ALU_ADD_HH;
3705
3706                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3707                 instr->alu.dst.n_bits = fdst->n_bits;
3708                 instr->alu.dst.offset = fdst->offset / 8;
3709                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3710                 instr->alu.src.n_bits = fsrc->n_bits;
3711                 instr->alu.src.offset = fsrc->offset / 8;
3712                 return 0;
3713         }
3714
3715         /* ADD_MI, ADD_HI. */
3716         src_val = strtoull(src, &src, 0);
3717         CHECK(!src[0], EINVAL);
3718
3719         instr->type = INSTR_ALU_ADD_MI;
3720         if (dst[0] == 'h')
3721                 instr->type = INSTR_ALU_ADD_HI;
3722
3723         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3724         instr->alu.dst.n_bits = fdst->n_bits;
3725         instr->alu.dst.offset = fdst->offset / 8;
3726         instr->alu.src_val = src_val;
3727         return 0;
3728 }
3729
3730 static int
3731 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3732                         struct action *action,
3733                         char **tokens,
3734                         int n_tokens,
3735                         struct instruction *instr,
3736                         struct instruction_data *data __rte_unused)
3737 {
3738         char *dst = tokens[1], *src = tokens[2];
3739         struct field *fdst, *fsrc;
3740         uint64_t src_val;
3741         uint32_t dst_struct_id, src_struct_id;
3742
3743         CHECK(n_tokens == 3, EINVAL);
3744
3745         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3746         CHECK(fdst, EINVAL);
3747
3748         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3749         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3750         if (fsrc) {
3751                 instr->type = INSTR_ALU_SUB;
3752                 if (dst[0] == 'h' && src[0] != 'h')
3753                         instr->type = INSTR_ALU_SUB_HM;
3754                 if (dst[0] != 'h' && src[0] == 'h')
3755                         instr->type = INSTR_ALU_SUB_MH;
3756                 if (dst[0] == 'h' && src[0] == 'h')
3757                         instr->type = INSTR_ALU_SUB_HH;
3758
3759                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3760                 instr->alu.dst.n_bits = fdst->n_bits;
3761                 instr->alu.dst.offset = fdst->offset / 8;
3762                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3763                 instr->alu.src.n_bits = fsrc->n_bits;
3764                 instr->alu.src.offset = fsrc->offset / 8;
3765                 return 0;
3766         }
3767
3768         /* SUB_MI, SUB_HI. */
3769         src_val = strtoull(src, &src, 0);
3770         CHECK(!src[0], EINVAL);
3771
3772         instr->type = INSTR_ALU_SUB_MI;
3773         if (dst[0] == 'h')
3774                 instr->type = INSTR_ALU_SUB_HI;
3775
3776         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3777         instr->alu.dst.n_bits = fdst->n_bits;
3778         instr->alu.dst.offset = fdst->offset / 8;
3779         instr->alu.src_val = src_val;
3780         return 0;
3781 }
3782
3783 static int
3784 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3785                           struct action *action __rte_unused,
3786                           char **tokens,
3787                           int n_tokens,
3788                           struct instruction *instr,
3789                           struct instruction_data *data __rte_unused)
3790 {
3791         char *dst = tokens[1], *src = tokens[2];
3792         struct header *hdst, *hsrc;
3793         struct field *fdst, *fsrc;
3794
3795         CHECK(n_tokens == 3, EINVAL);
3796
3797         fdst = header_field_parse(p, dst, &hdst);
3798         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3799
3800         /* CKADD_FIELD. */
3801         fsrc = header_field_parse(p, src, &hsrc);
3802         if (fsrc) {
3803                 instr->type = INSTR_ALU_CKADD_FIELD;
3804                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3805                 instr->alu.dst.n_bits = fdst->n_bits;
3806                 instr->alu.dst.offset = fdst->offset / 8;
3807                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3808                 instr->alu.src.n_bits = fsrc->n_bits;
3809                 instr->alu.src.offset = fsrc->offset / 8;
3810                 return 0;
3811         }
3812
3813         /* CKADD_STRUCT, CKADD_STRUCT20. */
3814         hsrc = header_parse(p, src);
3815         CHECK(hsrc, EINVAL);
3816
3817         instr->type = INSTR_ALU_CKADD_STRUCT;
3818         if ((hsrc->st->n_bits / 8) == 20)
3819                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3820
3821         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3822         instr->alu.dst.n_bits = fdst->n_bits;
3823         instr->alu.dst.offset = fdst->offset / 8;
3824         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3825         instr->alu.src.n_bits = hsrc->st->n_bits;
3826         instr->alu.src.offset = 0; /* Unused. */
3827         return 0;
3828 }
3829
3830 static int
3831 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3832                           struct action *action __rte_unused,
3833                           char **tokens,
3834                           int n_tokens,
3835                           struct instruction *instr,
3836                           struct instruction_data *data __rte_unused)
3837 {
3838         char *dst = tokens[1], *src = tokens[2];
3839         struct header *hdst, *hsrc;
3840         struct field *fdst, *fsrc;
3841
3842         CHECK(n_tokens == 3, EINVAL);
3843
3844         fdst = header_field_parse(p, dst, &hdst);
3845         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3846
3847         fsrc = header_field_parse(p, src, &hsrc);
3848         CHECK(fsrc, EINVAL);
3849
3850         instr->type = INSTR_ALU_CKSUB_FIELD;
3851         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3852         instr->alu.dst.n_bits = fdst->n_bits;
3853         instr->alu.dst.offset = fdst->offset / 8;
3854         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3855         instr->alu.src.n_bits = fsrc->n_bits;
3856         instr->alu.src.offset = fsrc->offset / 8;
3857         return 0;
3858 }
3859
3860 static int
3861 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3862                         struct action *action,
3863                         char **tokens,
3864                         int n_tokens,
3865                         struct instruction *instr,
3866                         struct instruction_data *data __rte_unused)
3867 {
3868         char *dst = tokens[1], *src = tokens[2];
3869         struct field *fdst, *fsrc;
3870         uint64_t src_val;
3871         uint32_t dst_struct_id, src_struct_id;
3872
3873         CHECK(n_tokens == 3, EINVAL);
3874
3875         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3876         CHECK(fdst, EINVAL);
3877
3878         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3879         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3880         if (fsrc) {
3881                 instr->type = INSTR_ALU_SHL;
3882                 if (dst[0] == 'h' && src[0] != 'h')
3883                         instr->type = INSTR_ALU_SHL_HM;
3884                 if (dst[0] != 'h' && src[0] == 'h')
3885                         instr->type = INSTR_ALU_SHL_MH;
3886                 if (dst[0] == 'h' && src[0] == 'h')
3887                         instr->type = INSTR_ALU_SHL_HH;
3888
3889                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3890                 instr->alu.dst.n_bits = fdst->n_bits;
3891                 instr->alu.dst.offset = fdst->offset / 8;
3892                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3893                 instr->alu.src.n_bits = fsrc->n_bits;
3894                 instr->alu.src.offset = fsrc->offset / 8;
3895                 return 0;
3896         }
3897
3898         /* SHL_MI, SHL_HI. */
3899         src_val = strtoull(src, &src, 0);
3900         CHECK(!src[0], EINVAL);
3901
3902         instr->type = INSTR_ALU_SHL_MI;
3903         if (dst[0] == 'h')
3904                 instr->type = INSTR_ALU_SHL_HI;
3905
3906         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3907         instr->alu.dst.n_bits = fdst->n_bits;
3908         instr->alu.dst.offset = fdst->offset / 8;
3909         instr->alu.src_val = src_val;
3910         return 0;
3911 }
3912
3913 static int
3914 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3915                         struct action *action,
3916                         char **tokens,
3917                         int n_tokens,
3918                         struct instruction *instr,
3919                         struct instruction_data *data __rte_unused)
3920 {
3921         char *dst = tokens[1], *src = tokens[2];
3922         struct field *fdst, *fsrc;
3923         uint64_t src_val;
3924         uint32_t dst_struct_id, src_struct_id;
3925
3926         CHECK(n_tokens == 3, EINVAL);
3927
3928         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3929         CHECK(fdst, EINVAL);
3930
3931         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3932         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3933         if (fsrc) {
3934                 instr->type = INSTR_ALU_SHR;
3935                 if (dst[0] == 'h' && src[0] != 'h')
3936                         instr->type = INSTR_ALU_SHR_HM;
3937                 if (dst[0] != 'h' && src[0] == 'h')
3938                         instr->type = INSTR_ALU_SHR_MH;
3939                 if (dst[0] == 'h' && src[0] == 'h')
3940                         instr->type = INSTR_ALU_SHR_HH;
3941
3942                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3943                 instr->alu.dst.n_bits = fdst->n_bits;
3944                 instr->alu.dst.offset = fdst->offset / 8;
3945                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3946                 instr->alu.src.n_bits = fsrc->n_bits;
3947                 instr->alu.src.offset = fsrc->offset / 8;
3948                 return 0;
3949         }
3950
3951         /* SHR_MI, SHR_HI. */
3952         src_val = strtoull(src, &src, 0);
3953         CHECK(!src[0], EINVAL);
3954
3955         instr->type = INSTR_ALU_SHR_MI;
3956         if (dst[0] == 'h')
3957                 instr->type = INSTR_ALU_SHR_HI;
3958
3959         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3960         instr->alu.dst.n_bits = fdst->n_bits;
3961         instr->alu.dst.offset = fdst->offset / 8;
3962         instr->alu.src_val = src_val;
3963         return 0;
3964 }
3965
3966 static int
3967 instr_alu_and_translate(struct rte_swx_pipeline *p,
3968                         struct action *action,
3969                         char **tokens,
3970                         int n_tokens,
3971                         struct instruction *instr,
3972                         struct instruction_data *data __rte_unused)
3973 {
3974         char *dst = tokens[1], *src = tokens[2];
3975         struct field *fdst, *fsrc;
3976         uint64_t src_val;
3977         uint32_t dst_struct_id, src_struct_id;
3978
3979         CHECK(n_tokens == 3, EINVAL);
3980
3981         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3982         CHECK(fdst, EINVAL);
3983
3984         /* AND or AND_S. */
3985         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3986         if (fsrc) {
3987                 instr->type = INSTR_ALU_AND;
3988                 if ((dst[0] == 'h' && src[0] != 'h') ||
3989                     (dst[0] != 'h' && src[0] == 'h'))
3990                         instr->type = INSTR_ALU_AND_S;
3991
3992                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3993                 instr->alu.dst.n_bits = fdst->n_bits;
3994                 instr->alu.dst.offset = fdst->offset / 8;
3995                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3996                 instr->alu.src.n_bits = fsrc->n_bits;
3997                 instr->alu.src.offset = fsrc->offset / 8;
3998                 return 0;
3999         }
4000
4001         /* AND_I. */
4002         src_val = strtoull(src, &src, 0);
4003         CHECK(!src[0], EINVAL);
4004
4005         if (dst[0] == 'h')
4006                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4007
4008         instr->type = INSTR_ALU_AND_I;
4009         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4010         instr->alu.dst.n_bits = fdst->n_bits;
4011         instr->alu.dst.offset = fdst->offset / 8;
4012         instr->alu.src_val = src_val;
4013         return 0;
4014 }
4015
4016 static int
4017 instr_alu_or_translate(struct rte_swx_pipeline *p,
4018                        struct action *action,
4019                        char **tokens,
4020                        int n_tokens,
4021                        struct instruction *instr,
4022                        struct instruction_data *data __rte_unused)
4023 {
4024         char *dst = tokens[1], *src = tokens[2];
4025         struct field *fdst, *fsrc;
4026         uint64_t src_val;
4027         uint32_t dst_struct_id, src_struct_id;
4028
4029         CHECK(n_tokens == 3, EINVAL);
4030
4031         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4032         CHECK(fdst, EINVAL);
4033
4034         /* OR or OR_S. */
4035         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4036         if (fsrc) {
4037                 instr->type = INSTR_ALU_OR;
4038                 if ((dst[0] == 'h' && src[0] != 'h') ||
4039                     (dst[0] != 'h' && src[0] == 'h'))
4040                         instr->type = INSTR_ALU_OR_S;
4041
4042                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4043                 instr->alu.dst.n_bits = fdst->n_bits;
4044                 instr->alu.dst.offset = fdst->offset / 8;
4045                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4046                 instr->alu.src.n_bits = fsrc->n_bits;
4047                 instr->alu.src.offset = fsrc->offset / 8;
4048                 return 0;
4049         }
4050
4051         /* OR_I. */
4052         src_val = strtoull(src, &src, 0);
4053         CHECK(!src[0], EINVAL);
4054
4055         if (dst[0] == 'h')
4056                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4057
4058         instr->type = INSTR_ALU_OR_I;
4059         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4060         instr->alu.dst.n_bits = fdst->n_bits;
4061         instr->alu.dst.offset = fdst->offset / 8;
4062         instr->alu.src_val = src_val;
4063         return 0;
4064 }
4065
4066 static int
4067 instr_alu_xor_translate(struct rte_swx_pipeline *p,
4068                         struct action *action,
4069                         char **tokens,
4070                         int n_tokens,
4071                         struct instruction *instr,
4072                         struct instruction_data *data __rte_unused)
4073 {
4074         char *dst = tokens[1], *src = tokens[2];
4075         struct field *fdst, *fsrc;
4076         uint64_t src_val;
4077         uint32_t dst_struct_id, src_struct_id;
4078
4079         CHECK(n_tokens == 3, EINVAL);
4080
4081         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4082         CHECK(fdst, EINVAL);
4083
4084         /* XOR or XOR_S. */
4085         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4086         if (fsrc) {
4087                 instr->type = INSTR_ALU_XOR;
4088                 if ((dst[0] == 'h' && src[0] != 'h') ||
4089                     (dst[0] != 'h' && src[0] == 'h'))
4090                         instr->type = INSTR_ALU_XOR_S;
4091
4092                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4093                 instr->alu.dst.n_bits = fdst->n_bits;
4094                 instr->alu.dst.offset = fdst->offset / 8;
4095                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4096                 instr->alu.src.n_bits = fsrc->n_bits;
4097                 instr->alu.src.offset = fsrc->offset / 8;
4098                 return 0;
4099         }
4100
4101         /* XOR_I. */
4102         src_val = strtoull(src, &src, 0);
4103         CHECK(!src[0], EINVAL);
4104
4105         if (dst[0] == 'h')
4106                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4107
4108         instr->type = INSTR_ALU_XOR_I;
4109         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4110         instr->alu.dst.n_bits = fdst->n_bits;
4111         instr->alu.dst.offset = fdst->offset / 8;
4112         instr->alu.src_val = src_val;
4113         return 0;
4114 }
4115
4116 static inline void
4117 instr_alu_add_exec(struct rte_swx_pipeline *p)
4118 {
4119         struct thread *t = &p->threads[p->thread_id];
4120         struct instruction *ip = t->ip;
4121
4122         TRACE("[Thread %2u] add\n", p->thread_id);
4123
4124         /* Structs. */
4125         ALU(t, ip, +);
4126
4127         /* Thread. */
4128         thread_ip_inc(p);
4129 }
4130
4131 static inline void
4132 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
4133 {
4134         struct thread *t = &p->threads[p->thread_id];
4135         struct instruction *ip = t->ip;
4136
4137         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
4138
4139         /* Structs. */
4140         ALU_MH(t, ip, +);
4141
4142         /* Thread. */
4143         thread_ip_inc(p);
4144 }
4145
4146 static inline void
4147 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
4148 {
4149         struct thread *t = &p->threads[p->thread_id];
4150         struct instruction *ip = t->ip;
4151
4152         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
4153
4154         /* Structs. */
4155         ALU_HM(t, ip, +);
4156
4157         /* Thread. */
4158         thread_ip_inc(p);
4159 }
4160
4161 static inline void
4162 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
4163 {
4164         struct thread *t = &p->threads[p->thread_id];
4165         struct instruction *ip = t->ip;
4166
4167         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
4168
4169         /* Structs. */
4170         ALU_HH(t, ip, +);
4171
4172         /* Thread. */
4173         thread_ip_inc(p);
4174 }
4175
4176 static inline void
4177 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
4178 {
4179         struct thread *t = &p->threads[p->thread_id];
4180         struct instruction *ip = t->ip;
4181
4182         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
4183
4184         /* Structs. */
4185         ALU_MI(t, ip, +);
4186
4187         /* Thread. */
4188         thread_ip_inc(p);
4189 }
4190
4191 static inline void
4192 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
4193 {
4194         struct thread *t = &p->threads[p->thread_id];
4195         struct instruction *ip = t->ip;
4196
4197         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
4198
4199         /* Structs. */
4200         ALU_HI(t, ip, +);
4201
4202         /* Thread. */
4203         thread_ip_inc(p);
4204 }
4205
4206 static inline void
4207 instr_alu_sub_exec(struct rte_swx_pipeline *p)
4208 {
4209         struct thread *t = &p->threads[p->thread_id];
4210         struct instruction *ip = t->ip;
4211
4212         TRACE("[Thread %2u] sub\n", p->thread_id);
4213
4214         /* Structs. */
4215         ALU(t, ip, -);
4216
4217         /* Thread. */
4218         thread_ip_inc(p);
4219 }
4220
4221 static inline void
4222 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
4223 {
4224         struct thread *t = &p->threads[p->thread_id];
4225         struct instruction *ip = t->ip;
4226
4227         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
4228
4229         /* Structs. */
4230         ALU_MH(t, ip, -);
4231
4232         /* Thread. */
4233         thread_ip_inc(p);
4234 }
4235
4236 static inline void
4237 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4238 {
4239         struct thread *t = &p->threads[p->thread_id];
4240         struct instruction *ip = t->ip;
4241
4242         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4243
4244         /* Structs. */
4245         ALU_HM(t, ip, -);
4246
4247         /* Thread. */
4248         thread_ip_inc(p);
4249 }
4250
4251 static inline void
4252 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4253 {
4254         struct thread *t = &p->threads[p->thread_id];
4255         struct instruction *ip = t->ip;
4256
4257         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4258
4259         /* Structs. */
4260         ALU_HH(t, ip, -);
4261
4262         /* Thread. */
4263         thread_ip_inc(p);
4264 }
4265
4266 static inline void
4267 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4268 {
4269         struct thread *t = &p->threads[p->thread_id];
4270         struct instruction *ip = t->ip;
4271
4272         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4273
4274         /* Structs. */
4275         ALU_MI(t, ip, -);
4276
4277         /* Thread. */
4278         thread_ip_inc(p);
4279 }
4280
4281 static inline void
4282 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4283 {
4284         struct thread *t = &p->threads[p->thread_id];
4285         struct instruction *ip = t->ip;
4286
4287         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4288
4289         /* Structs. */
4290         ALU_HI(t, ip, -);
4291
4292         /* Thread. */
4293         thread_ip_inc(p);
4294 }
4295
4296 static inline void
4297 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4298 {
4299         struct thread *t = &p->threads[p->thread_id];
4300         struct instruction *ip = t->ip;
4301
4302         TRACE("[Thread %2u] shl\n", p->thread_id);
4303
4304         /* Structs. */
4305         ALU(t, ip, <<);
4306
4307         /* Thread. */
4308         thread_ip_inc(p);
4309 }
4310
4311 static inline void
4312 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4313 {
4314         struct thread *t = &p->threads[p->thread_id];
4315         struct instruction *ip = t->ip;
4316
4317         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4318
4319         /* Structs. */
4320         ALU_MH(t, ip, <<);
4321
4322         /* Thread. */
4323         thread_ip_inc(p);
4324 }
4325
4326 static inline void
4327 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4328 {
4329         struct thread *t = &p->threads[p->thread_id];
4330         struct instruction *ip = t->ip;
4331
4332         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4333
4334         /* Structs. */
4335         ALU_HM(t, ip, <<);
4336
4337         /* Thread. */
4338         thread_ip_inc(p);
4339 }
4340
4341 static inline void
4342 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4343 {
4344         struct thread *t = &p->threads[p->thread_id];
4345         struct instruction *ip = t->ip;
4346
4347         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4348
4349         /* Structs. */
4350         ALU_HH(t, ip, <<);
4351
4352         /* Thread. */
4353         thread_ip_inc(p);
4354 }
4355
4356 static inline void
4357 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4358 {
4359         struct thread *t = &p->threads[p->thread_id];
4360         struct instruction *ip = t->ip;
4361
4362         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4363
4364         /* Structs. */
4365         ALU_MI(t, ip, <<);
4366
4367         /* Thread. */
4368         thread_ip_inc(p);
4369 }
4370
4371 static inline void
4372 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4373 {
4374         struct thread *t = &p->threads[p->thread_id];
4375         struct instruction *ip = t->ip;
4376
4377         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4378
4379         /* Structs. */
4380         ALU_HI(t, ip, <<);
4381
4382         /* Thread. */
4383         thread_ip_inc(p);
4384 }
4385
4386 static inline void
4387 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4388 {
4389         struct thread *t = &p->threads[p->thread_id];
4390         struct instruction *ip = t->ip;
4391
4392         TRACE("[Thread %2u] shr\n", p->thread_id);
4393
4394         /* Structs. */
4395         ALU(t, ip, >>);
4396
4397         /* Thread. */
4398         thread_ip_inc(p);
4399 }
4400
4401 static inline void
4402 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4403 {
4404         struct thread *t = &p->threads[p->thread_id];
4405         struct instruction *ip = t->ip;
4406
4407         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4408
4409         /* Structs. */
4410         ALU_MH(t, ip, >>);
4411
4412         /* Thread. */
4413         thread_ip_inc(p);
4414 }
4415
4416 static inline void
4417 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4418 {
4419         struct thread *t = &p->threads[p->thread_id];
4420         struct instruction *ip = t->ip;
4421
4422         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4423
4424         /* Structs. */
4425         ALU_HM(t, ip, >>);
4426
4427         /* Thread. */
4428         thread_ip_inc(p);
4429 }
4430
4431 static inline void
4432 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4433 {
4434         struct thread *t = &p->threads[p->thread_id];
4435         struct instruction *ip = t->ip;
4436
4437         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4438
4439         /* Structs. */
4440         ALU_HH(t, ip, >>);
4441
4442         /* Thread. */
4443         thread_ip_inc(p);
4444 }
4445
4446 static inline void
4447 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4448 {
4449         struct thread *t = &p->threads[p->thread_id];
4450         struct instruction *ip = t->ip;
4451
4452         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4453
4454         /* Structs. */
4455         ALU_MI(t, ip, >>);
4456
4457         /* Thread. */
4458         thread_ip_inc(p);
4459 }
4460
4461 static inline void
4462 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4463 {
4464         struct thread *t = &p->threads[p->thread_id];
4465         struct instruction *ip = t->ip;
4466
4467         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4468
4469         /* Structs. */
4470         ALU_HI(t, ip, >>);
4471
4472         /* Thread. */
4473         thread_ip_inc(p);
4474 }
4475
4476 static inline void
4477 instr_alu_and_exec(struct rte_swx_pipeline *p)
4478 {
4479         struct thread *t = &p->threads[p->thread_id];
4480         struct instruction *ip = t->ip;
4481
4482         TRACE("[Thread %2u] and\n", p->thread_id);
4483
4484         /* Structs. */
4485         ALU(t, ip, &);
4486
4487         /* Thread. */
4488         thread_ip_inc(p);
4489 }
4490
4491 static inline void
4492 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
4493 {
4494         struct thread *t = &p->threads[p->thread_id];
4495         struct instruction *ip = t->ip;
4496
4497         TRACE("[Thread %2u] and (s)\n", p->thread_id);
4498
4499         /* Structs. */
4500         ALU_S(t, ip, &);
4501
4502         /* Thread. */
4503         thread_ip_inc(p);
4504 }
4505
4506 static inline void
4507 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4508 {
4509         struct thread *t = &p->threads[p->thread_id];
4510         struct instruction *ip = t->ip;
4511
4512         TRACE("[Thread %2u] and (i)\n", p->thread_id);
4513
4514         /* Structs. */
4515         ALU_I(t, ip, &);
4516
4517         /* Thread. */
4518         thread_ip_inc(p);
4519 }
4520
4521 static inline void
4522 instr_alu_or_exec(struct rte_swx_pipeline *p)
4523 {
4524         struct thread *t = &p->threads[p->thread_id];
4525         struct instruction *ip = t->ip;
4526
4527         TRACE("[Thread %2u] or\n", p->thread_id);
4528
4529         /* Structs. */
4530         ALU(t, ip, |);
4531
4532         /* Thread. */
4533         thread_ip_inc(p);
4534 }
4535
4536 static inline void
4537 instr_alu_or_s_exec(struct rte_swx_pipeline *p)
4538 {
4539         struct thread *t = &p->threads[p->thread_id];
4540         struct instruction *ip = t->ip;
4541
4542         TRACE("[Thread %2u] or (s)\n", p->thread_id);
4543
4544         /* Structs. */
4545         ALU_S(t, ip, |);
4546
4547         /* Thread. */
4548         thread_ip_inc(p);
4549 }
4550
4551 static inline void
4552 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4553 {
4554         struct thread *t = &p->threads[p->thread_id];
4555         struct instruction *ip = t->ip;
4556
4557         TRACE("[Thread %2u] or (i)\n", p->thread_id);
4558
4559         /* Structs. */
4560         ALU_I(t, ip, |);
4561
4562         /* Thread. */
4563         thread_ip_inc(p);
4564 }
4565
4566 static inline void
4567 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4568 {
4569         struct thread *t = &p->threads[p->thread_id];
4570         struct instruction *ip = t->ip;
4571
4572         TRACE("[Thread %2u] xor\n", p->thread_id);
4573
4574         /* Structs. */
4575         ALU(t, ip, ^);
4576
4577         /* Thread. */
4578         thread_ip_inc(p);
4579 }
4580
4581 static inline void
4582 instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
4583 {
4584         struct thread *t = &p->threads[p->thread_id];
4585         struct instruction *ip = t->ip;
4586
4587         TRACE("[Thread %2u] xor (s)\n", p->thread_id);
4588
4589         /* Structs. */
4590         ALU_S(t, ip, ^);
4591
4592         /* Thread. */
4593         thread_ip_inc(p);
4594 }
4595
4596 static inline void
4597 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4598 {
4599         struct thread *t = &p->threads[p->thread_id];
4600         struct instruction *ip = t->ip;
4601
4602         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
4603
4604         /* Structs. */
4605         ALU_I(t, ip, ^);
4606
4607         /* Thread. */
4608         thread_ip_inc(p);
4609 }
4610
4611 static inline void
4612 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4613 {
4614         struct thread *t = &p->threads[p->thread_id];
4615         struct instruction *ip = t->ip;
4616         uint8_t *dst_struct, *src_struct;
4617         uint16_t *dst16_ptr, dst;
4618         uint64_t *src64_ptr, src64, src64_mask, src;
4619         uint64_t r;
4620
4621         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
4622
4623         /* Structs. */
4624         dst_struct = t->structs[ip->alu.dst.struct_id];
4625         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4626         dst = *dst16_ptr;
4627
4628         src_struct = t->structs[ip->alu.src.struct_id];
4629         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4630         src64 = *src64_ptr;
4631         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4632         src = src64 & src64_mask;
4633
4634         r = dst;
4635         r = ~r & 0xFFFF;
4636
4637         /* The first input (r) is a 16-bit number. The second and the third
4638          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
4639          * three numbers (output r) is a 34-bit number.
4640          */
4641         r += (src >> 32) + (src & 0xFFFFFFFF);
4642
4643         /* The first input is a 16-bit number. The second input is an 18-bit
4644          * number. In the worst case scenario, the sum of the two numbers is a
4645          * 19-bit number.
4646          */
4647         r = (r & 0xFFFF) + (r >> 16);
4648
4649         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4650          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
4651          */
4652         r = (r & 0xFFFF) + (r >> 16);
4653
4654         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4655          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4656          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
4657          * therefore the output r is always a 16-bit number.
4658          */
4659         r = (r & 0xFFFF) + (r >> 16);
4660
4661         r = ~r & 0xFFFF;
4662         r = r ? r : 0xFFFF;
4663
4664         *dst16_ptr = (uint16_t)r;
4665
4666         /* Thread. */
4667         thread_ip_inc(p);
4668 }
4669
4670 static inline void
4671 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4672 {
4673         struct thread *t = &p->threads[p->thread_id];
4674         struct instruction *ip = t->ip;
4675         uint8_t *dst_struct, *src_struct;
4676         uint16_t *dst16_ptr, dst;
4677         uint64_t *src64_ptr, src64, src64_mask, src;
4678         uint64_t r;
4679
4680         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
4681
4682         /* Structs. */
4683         dst_struct = t->structs[ip->alu.dst.struct_id];
4684         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4685         dst = *dst16_ptr;
4686
4687         src_struct = t->structs[ip->alu.src.struct_id];
4688         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4689         src64 = *src64_ptr;
4690         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4691         src = src64 & src64_mask;
4692
4693         r = dst;
4694         r = ~r & 0xFFFF;
4695
4696         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
4697          * the following sequence of operations in 2's complement arithmetic:
4698          *    a '- b = (a - b) % 0xFFFF.
4699          *
4700          * In order to prevent an underflow for the below subtraction, in which
4701          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
4702          * minuend), we first add a multiple of the 0xFFFF modulus to the
4703          * minuend. The number we add to the minuend needs to be a 34-bit number
4704          * or higher, so for readability reasons we picked the 36-bit multiple.
4705          * We are effectively turning the 16-bit minuend into a 36-bit number:
4706          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
4707          */
4708         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
4709
4710         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
4711          * result (the output r) is a 36-bit number.
4712          */
4713         r -= (src >> 32) + (src & 0xFFFFFFFF);
4714
4715         /* The first input is a 16-bit number. The second input is a 20-bit
4716          * number. Their sum is a 21-bit number.
4717          */
4718         r = (r & 0xFFFF) + (r >> 16);
4719
4720         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4721          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
4722          */
4723         r = (r & 0xFFFF) + (r >> 16);
4724
4725         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4726          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4727          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4728          * generated, therefore the output r is always a 16-bit number.
4729          */
4730         r = (r & 0xFFFF) + (r >> 16);
4731
4732         r = ~r & 0xFFFF;
4733         r = r ? r : 0xFFFF;
4734
4735         *dst16_ptr = (uint16_t)r;
4736
4737         /* Thread. */
4738         thread_ip_inc(p);
4739 }
4740
4741 static inline void
4742 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4743 {
4744         struct thread *t = &p->threads[p->thread_id];
4745         struct instruction *ip = t->ip;
4746         uint8_t *dst_struct, *src_struct;
4747         uint16_t *dst16_ptr;
4748         uint32_t *src32_ptr;
4749         uint64_t r0, r1;
4750
4751         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
4752
4753         /* Structs. */
4754         dst_struct = t->structs[ip->alu.dst.struct_id];
4755         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4756
4757         src_struct = t->structs[ip->alu.src.struct_id];
4758         src32_ptr = (uint32_t *)&src_struct[0];
4759
4760         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
4761         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
4762         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
4763         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
4764         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
4765
4766         /* The first input is a 16-bit number. The second input is a 19-bit
4767          * number. Their sum is a 20-bit number.
4768          */
4769         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4770
4771         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4772          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
4773          */
4774         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4775
4776         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4777          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4778          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
4779          * generated, therefore the output r is always a 16-bit number.
4780          */
4781         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4782
4783         r0 = ~r0 & 0xFFFF;
4784         r0 = r0 ? r0 : 0xFFFF;
4785
4786         *dst16_ptr = (uint16_t)r0;
4787
4788         /* Thread. */
4789         thread_ip_inc(p);
4790 }
4791
4792 static inline void
4793 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4794 {
4795         struct thread *t = &p->threads[p->thread_id];
4796         struct instruction *ip = t->ip;
4797         uint8_t *dst_struct, *src_struct;
4798         uint16_t *dst16_ptr;
4799         uint32_t *src32_ptr;
4800         uint64_t r = 0;
4801         uint32_t i;
4802
4803         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
4804
4805         /* Structs. */
4806         dst_struct = t->structs[ip->alu.dst.struct_id];
4807         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4808
4809         src_struct = t->structs[ip->alu.src.struct_id];
4810         src32_ptr = (uint32_t *)&src_struct[0];
4811
4812         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
4813          * Therefore, in the worst case scenario, a 35-bit number is added to a
4814          * 16-bit number (the input r), so the output r is 36-bit number.
4815          */
4816         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
4817                 r += *src32_ptr;
4818
4819         /* The first input is a 16-bit number. The second input is a 20-bit
4820          * number. Their sum is a 21-bit number.
4821          */
4822         r = (r & 0xFFFF) + (r >> 16);
4823
4824         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4825          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
4826          */
4827         r = (r & 0xFFFF) + (r >> 16);
4828
4829         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4830          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4831          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4832          * generated, therefore the output r is always a 16-bit number.
4833          */
4834         r = (r & 0xFFFF) + (r >> 16);
4835
4836         r = ~r & 0xFFFF;
4837         r = r ? r : 0xFFFF;
4838
4839         *dst16_ptr = (uint16_t)r;
4840
4841         /* Thread. */
4842         thread_ip_inc(p);
4843 }
4844
4845 /*
4846  * Register array.
4847  */
4848 static struct regarray *
4849 regarray_find(struct rte_swx_pipeline *p, const char *name);
4850
4851 static int
4852 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4853                       struct action *action,
4854                       char **tokens,
4855                       int n_tokens,
4856                       struct instruction *instr,
4857                       struct instruction_data *data __rte_unused)
4858 {
4859         char *regarray = tokens[1], *idx = tokens[2];
4860         struct regarray *r;
4861         struct field *fidx;
4862         uint32_t idx_struct_id, idx_val;
4863
4864         CHECK(n_tokens == 3, EINVAL);
4865
4866         r = regarray_find(p, regarray);
4867         CHECK(r, EINVAL);
4868
4869         /* REGPREFETCH_RH, REGPREFETCH_RM. */
4870         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4871         if (fidx) {
4872                 instr->type = INSTR_REGPREFETCH_RM;
4873                 if (idx[0] == 'h')
4874                         instr->type = INSTR_REGPREFETCH_RH;
4875
4876                 instr->regarray.regarray_id = r->id;
4877                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4878                 instr->regarray.idx.n_bits = fidx->n_bits;
4879                 instr->regarray.idx.offset = fidx->offset / 8;
4880                 instr->regarray.dstsrc_val = 0; /* Unused. */
4881                 return 0;
4882         }
4883
4884         /* REGPREFETCH_RI. */
4885         idx_val = strtoul(idx, &idx, 0);
4886         CHECK(!idx[0], EINVAL);
4887
4888         instr->type = INSTR_REGPREFETCH_RI;
4889         instr->regarray.regarray_id = r->id;
4890         instr->regarray.idx_val = idx_val;
4891         instr->regarray.dstsrc_val = 0; /* Unused. */
4892         return 0;
4893 }
4894
4895 static int
4896 instr_regrd_translate(struct rte_swx_pipeline *p,
4897                       struct action *action,
4898                       char **tokens,
4899                       int n_tokens,
4900                       struct instruction *instr,
4901                       struct instruction_data *data __rte_unused)
4902 {
4903         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4904         struct regarray *r;
4905         struct field *fdst, *fidx;
4906         uint32_t dst_struct_id, idx_struct_id, idx_val;
4907
4908         CHECK(n_tokens == 4, EINVAL);
4909
4910         r = regarray_find(p, regarray);
4911         CHECK(r, EINVAL);
4912
4913         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4914         CHECK(fdst, EINVAL);
4915
4916         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4917         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4918         if (fidx) {
4919                 instr->type = INSTR_REGRD_MRM;
4920                 if (dst[0] == 'h' && idx[0] != 'h')
4921                         instr->type = INSTR_REGRD_HRM;
4922                 if (dst[0] != 'h' && idx[0] == 'h')
4923                         instr->type = INSTR_REGRD_MRH;
4924                 if (dst[0] == 'h' && idx[0] == 'h')
4925                         instr->type = INSTR_REGRD_HRH;
4926
4927                 instr->regarray.regarray_id = r->id;
4928                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4929                 instr->regarray.idx.n_bits = fidx->n_bits;
4930                 instr->regarray.idx.offset = fidx->offset / 8;
4931                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4932                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4933                 instr->regarray.dstsrc.offset = fdst->offset / 8;
4934                 return 0;
4935         }
4936
4937         /* REGRD_MRI, REGRD_HRI. */
4938         idx_val = strtoul(idx, &idx, 0);
4939         CHECK(!idx[0], EINVAL);
4940
4941         instr->type = INSTR_REGRD_MRI;
4942         if (dst[0] == 'h')
4943                 instr->type = INSTR_REGRD_HRI;
4944
4945         instr->regarray.regarray_id = r->id;
4946         instr->regarray.idx_val = idx_val;
4947         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4948         instr->regarray.dstsrc.n_bits = fdst->n_bits;
4949         instr->regarray.dstsrc.offset = fdst->offset / 8;
4950         return 0;
4951 }
4952
4953 static int
4954 instr_regwr_translate(struct rte_swx_pipeline *p,
4955                       struct action *action,
4956                       char **tokens,
4957                       int n_tokens,
4958                       struct instruction *instr,
4959                       struct instruction_data *data __rte_unused)
4960 {
4961         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4962         struct regarray *r;
4963         struct field *fidx, *fsrc;
4964         uint64_t src_val;
4965         uint32_t idx_struct_id, idx_val, src_struct_id;
4966
4967         CHECK(n_tokens == 4, EINVAL);
4968
4969         r = regarray_find(p, regarray);
4970         CHECK(r, EINVAL);
4971
4972         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4973         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4974         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4975         if (fidx && fsrc) {
4976                 instr->type = INSTR_REGWR_RMM;
4977                 if (idx[0] == 'h' && src[0] != 'h')
4978                         instr->type = INSTR_REGWR_RHM;
4979                 if (idx[0] != 'h' && src[0] == 'h')
4980                         instr->type = INSTR_REGWR_RMH;
4981                 if (idx[0] == 'h' && src[0] == 'h')
4982                         instr->type = INSTR_REGWR_RHH;
4983
4984                 instr->regarray.regarray_id = r->id;
4985                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4986                 instr->regarray.idx.n_bits = fidx->n_bits;
4987                 instr->regarray.idx.offset = fidx->offset / 8;
4988                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4989                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4990                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4991                 return 0;
4992         }
4993
4994         /* REGWR_RHI, REGWR_RMI. */
4995         if (fidx && !fsrc) {
4996                 src_val = strtoull(src, &src, 0);
4997                 CHECK(!src[0], EINVAL);
4998
4999                 instr->type = INSTR_REGWR_RMI;
5000                 if (idx[0] == 'h')
5001                         instr->type = INSTR_REGWR_RHI;
5002
5003                 instr->regarray.regarray_id = r->id;
5004                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5005                 instr->regarray.idx.n_bits = fidx->n_bits;
5006                 instr->regarray.idx.offset = fidx->offset / 8;
5007                 instr->regarray.dstsrc_val = src_val;
5008                 return 0;
5009         }
5010
5011         /* REGWR_RIH, REGWR_RIM. */
5012         if (!fidx && fsrc) {
5013                 idx_val = strtoul(idx, &idx, 0);
5014                 CHECK(!idx[0], EINVAL);
5015
5016                 instr->type = INSTR_REGWR_RIM;
5017                 if (src[0] == 'h')
5018                         instr->type = INSTR_REGWR_RIH;
5019
5020                 instr->regarray.regarray_id = r->id;
5021                 instr->regarray.idx_val = idx_val;
5022                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5023                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5024                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5025                 return 0;
5026         }
5027
5028         /* REGWR_RII. */
5029         src_val = strtoull(src, &src, 0);
5030         CHECK(!src[0], EINVAL);
5031
5032         idx_val = strtoul(idx, &idx, 0);
5033         CHECK(!idx[0], EINVAL);
5034
5035         instr->type = INSTR_REGWR_RII;
5036         instr->regarray.idx_val = idx_val;
5037         instr->regarray.dstsrc_val = src_val;
5038
5039         return 0;
5040 }
5041
5042 static int
5043 instr_regadd_translate(struct rte_swx_pipeline *p,
5044                        struct action *action,
5045                        char **tokens,
5046                        int n_tokens,
5047                        struct instruction *instr,
5048                        struct instruction_data *data __rte_unused)
5049 {
5050         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5051         struct regarray *r;
5052         struct field *fidx, *fsrc;
5053         uint64_t src_val;
5054         uint32_t idx_struct_id, idx_val, src_struct_id;
5055
5056         CHECK(n_tokens == 4, EINVAL);
5057
5058         r = regarray_find(p, regarray);
5059         CHECK(r, EINVAL);
5060
5061         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
5062         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5063         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5064         if (fidx && fsrc) {
5065                 instr->type = INSTR_REGADD_RMM;
5066                 if (idx[0] == 'h' && src[0] != 'h')
5067                         instr->type = INSTR_REGADD_RHM;
5068                 if (idx[0] != 'h' && src[0] == 'h')
5069                         instr->type = INSTR_REGADD_RMH;
5070                 if (idx[0] == 'h' && src[0] == 'h')
5071                         instr->type = INSTR_REGADD_RHH;
5072
5073                 instr->regarray.regarray_id = r->id;
5074                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5075                 instr->regarray.idx.n_bits = fidx->n_bits;
5076                 instr->regarray.idx.offset = fidx->offset / 8;
5077                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5078                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5079                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5080                 return 0;
5081         }
5082
5083         /* REGADD_RHI, REGADD_RMI. */
5084         if (fidx && !fsrc) {
5085                 src_val = strtoull(src, &src, 0);
5086                 CHECK(!src[0], EINVAL);
5087
5088                 instr->type = INSTR_REGADD_RMI;
5089                 if (idx[0] == 'h')
5090                         instr->type = INSTR_REGADD_RHI;
5091
5092                 instr->regarray.regarray_id = r->id;
5093                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5094                 instr->regarray.idx.n_bits = fidx->n_bits;
5095                 instr->regarray.idx.offset = fidx->offset / 8;
5096                 instr->regarray.dstsrc_val = src_val;
5097                 return 0;
5098         }
5099
5100         /* REGADD_RIH, REGADD_RIM. */
5101         if (!fidx && fsrc) {
5102                 idx_val = strtoul(idx, &idx, 0);
5103                 CHECK(!idx[0], EINVAL);
5104
5105                 instr->type = INSTR_REGADD_RIM;
5106                 if (src[0] == 'h')
5107                         instr->type = INSTR_REGADD_RIH;
5108
5109                 instr->regarray.regarray_id = r->id;
5110                 instr->regarray.idx_val = idx_val;
5111                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5112                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5113                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5114                 return 0;
5115         }
5116
5117         /* REGADD_RII. */
5118         src_val = strtoull(src, &src, 0);
5119         CHECK(!src[0], EINVAL);
5120
5121         idx_val = strtoul(idx, &idx, 0);
5122         CHECK(!idx[0], EINVAL);
5123
5124         instr->type = INSTR_REGADD_RII;
5125         instr->regarray.idx_val = idx_val;
5126         instr->regarray.dstsrc_val = src_val;
5127         return 0;
5128 }
5129
5130 static inline uint64_t *
5131 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
5132 {
5133         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5134         return r->regarray;
5135 }
5136
5137 static inline uint64_t
5138 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5139 {
5140         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5141
5142         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5143         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5144         uint64_t idx64 = *idx64_ptr;
5145         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
5146         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5147
5148         return idx;
5149 }
5150
5151 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5152
5153 static inline uint64_t
5154 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5155 {
5156         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5157
5158         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5159         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5160         uint64_t idx64 = *idx64_ptr;
5161         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
5162
5163         return idx;
5164 }
5165
5166 #else
5167
5168 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
5169
5170 #endif
5171
5172 static inline uint64_t
5173 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5174 {
5175         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5176
5177         uint64_t idx = ip->regarray.idx_val & r->size_mask;
5178
5179         return idx;
5180 }
5181
5182 static inline uint64_t
5183 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
5184 {
5185         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5186         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5187         uint64_t src64 = *src64_ptr;
5188         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5189         uint64_t src = src64 & src64_mask;
5190
5191         return src;
5192 }
5193
5194 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5195
5196 static inline uint64_t
5197 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
5198 {
5199         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5200         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5201         uint64_t src64 = *src64_ptr;
5202         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
5203
5204         return src;
5205 }
5206
5207 #else
5208
5209 #define instr_regarray_src_nbo instr_regarray_src_hbo
5210
5211 #endif
5212
5213 static inline void
5214 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5215 {
5216         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5217         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5218         uint64_t dst64 = *dst64_ptr;
5219         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5220
5221         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5222
5223 }
5224
5225 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5226
5227 static inline void
5228 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5229 {
5230         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5231         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5232         uint64_t dst64 = *dst64_ptr;
5233         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5234
5235         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
5236         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5237 }
5238
5239 #else
5240
5241 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
5242
5243 #endif
5244
5245 static inline void
5246 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
5247 {
5248         struct thread *t = &p->threads[p->thread_id];
5249         struct instruction *ip = t->ip;
5250         uint64_t *regarray, idx;
5251
5252         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
5253
5254         /* Structs. */
5255         regarray = instr_regarray_regarray(p, ip);
5256         idx = instr_regarray_idx_nbo(p, t, ip);
5257         rte_prefetch0(&regarray[idx]);
5258
5259         /* Thread. */
5260         thread_ip_inc(p);
5261 }
5262
5263 static inline void
5264 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
5265 {
5266         struct thread *t = &p->threads[p->thread_id];
5267         struct instruction *ip = t->ip;
5268         uint64_t *regarray, idx;
5269
5270         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
5271
5272         /* Structs. */
5273         regarray = instr_regarray_regarray(p, ip);
5274         idx = instr_regarray_idx_hbo(p, t, ip);
5275         rte_prefetch0(&regarray[idx]);
5276
5277         /* Thread. */
5278         thread_ip_inc(p);
5279 }
5280
5281 static inline void
5282 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
5283 {
5284         struct thread *t = &p->threads[p->thread_id];
5285         struct instruction *ip = t->ip;
5286         uint64_t *regarray, idx;
5287
5288         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
5289
5290         /* Structs. */
5291         regarray = instr_regarray_regarray(p, ip);
5292         idx = instr_regarray_idx_imm(p, ip);
5293         rte_prefetch0(&regarray[idx]);
5294
5295         /* Thread. */
5296         thread_ip_inc(p);
5297 }
5298
5299 static inline void
5300 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
5301 {
5302         struct thread *t = &p->threads[p->thread_id];
5303         struct instruction *ip = t->ip;
5304         uint64_t *regarray, idx;
5305
5306         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
5307
5308         /* Structs. */
5309         regarray = instr_regarray_regarray(p, ip);
5310         idx = instr_regarray_idx_nbo(p, t, ip);
5311         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5312
5313         /* Thread. */
5314         thread_ip_inc(p);
5315 }
5316
5317 static inline void
5318 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
5319 {
5320         struct thread *t = &p->threads[p->thread_id];
5321         struct instruction *ip = t->ip;
5322         uint64_t *regarray, idx;
5323
5324         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
5325
5326         /* Structs. */
5327         regarray = instr_regarray_regarray(p, ip);
5328         idx = instr_regarray_idx_hbo(p, t, ip);
5329         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5330
5331         /* Thread. */
5332         thread_ip_inc(p);
5333 }
5334
5335 static inline void
5336 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
5337 {
5338         struct thread *t = &p->threads[p->thread_id];
5339         struct instruction *ip = t->ip;
5340         uint64_t *regarray, idx;
5341
5342         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
5343
5344         /* Structs. */
5345         regarray = instr_regarray_regarray(p, ip);
5346         idx = instr_regarray_idx_nbo(p, t, ip);
5347         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5348
5349         /* Thread. */
5350         thread_ip_inc(p);
5351 }
5352
5353 static inline void
5354 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
5355 {
5356         struct thread *t = &p->threads[p->thread_id];
5357         struct instruction *ip = t->ip;
5358         uint64_t *regarray, idx;
5359
5360         /* Structs. */
5361         regarray = instr_regarray_regarray(p, ip);
5362         idx = instr_regarray_idx_hbo(p, t, ip);
5363         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5364
5365         /* Thread. */
5366         thread_ip_inc(p);
5367 }
5368
5369 static inline void
5370 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
5371 {
5372         struct thread *t = &p->threads[p->thread_id];
5373         struct instruction *ip = t->ip;
5374         uint64_t *regarray, idx;
5375
5376         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
5377
5378         /* Structs. */
5379         regarray = instr_regarray_regarray(p, ip);
5380         idx = instr_regarray_idx_imm(p, ip);
5381         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5382
5383         /* Thread. */
5384         thread_ip_inc(p);
5385 }
5386
5387 static inline void
5388 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
5389 {
5390         struct thread *t = &p->threads[p->thread_id];
5391         struct instruction *ip = t->ip;
5392         uint64_t *regarray, idx;
5393
5394         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
5395
5396         /* Structs. */
5397         regarray = instr_regarray_regarray(p, ip);
5398         idx = instr_regarray_idx_imm(p, ip);
5399         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5400
5401         /* Thread. */
5402         thread_ip_inc(p);
5403 }
5404
5405 static inline void
5406 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
5407 {
5408         struct thread *t = &p->threads[p->thread_id];
5409         struct instruction *ip = t->ip;
5410         uint64_t *regarray, idx, src;
5411
5412         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
5413
5414         /* Structs. */
5415         regarray = instr_regarray_regarray(p, ip);
5416         idx = instr_regarray_idx_nbo(p, t, ip);
5417         src = instr_regarray_src_nbo(t, ip);
5418         regarray[idx] = src;
5419
5420         /* Thread. */
5421         thread_ip_inc(p);
5422 }
5423
5424 static inline void
5425 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
5426 {
5427         struct thread *t = &p->threads[p->thread_id];
5428         struct instruction *ip = t->ip;
5429         uint64_t *regarray, idx, src;
5430
5431         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
5432
5433         /* Structs. */
5434         regarray = instr_regarray_regarray(p, ip);
5435         idx = instr_regarray_idx_nbo(p, t, ip);
5436         src = instr_regarray_src_hbo(t, ip);
5437         regarray[idx] = src;
5438
5439         /* Thread. */
5440         thread_ip_inc(p);
5441 }
5442
5443 static inline void
5444 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
5445 {
5446         struct thread *t = &p->threads[p->thread_id];
5447         struct instruction *ip = t->ip;
5448         uint64_t *regarray, idx, src;
5449
5450         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
5451
5452         /* Structs. */
5453         regarray = instr_regarray_regarray(p, ip);
5454         idx = instr_regarray_idx_hbo(p, t, ip);
5455         src = instr_regarray_src_nbo(t, ip);
5456         regarray[idx] = src;
5457
5458         /* Thread. */
5459         thread_ip_inc(p);
5460 }
5461
5462 static inline void
5463 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
5464 {
5465         struct thread *t = &p->threads[p->thread_id];
5466         struct instruction *ip = t->ip;
5467         uint64_t *regarray, idx, src;
5468
5469         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
5470
5471         /* Structs. */
5472         regarray = instr_regarray_regarray(p, ip);
5473         idx = instr_regarray_idx_hbo(p, t, ip);
5474         src = instr_regarray_src_hbo(t, ip);
5475         regarray[idx] = src;
5476
5477         /* Thread. */
5478         thread_ip_inc(p);
5479 }
5480
5481 static inline void
5482 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
5483 {
5484         struct thread *t = &p->threads[p->thread_id];
5485         struct instruction *ip = t->ip;
5486         uint64_t *regarray, idx, src;
5487
5488         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
5489
5490         /* Structs. */
5491         regarray = instr_regarray_regarray(p, ip);
5492         idx = instr_regarray_idx_nbo(p, t, ip);
5493         src = ip->regarray.dstsrc_val;
5494         regarray[idx] = src;
5495
5496         /* Thread. */
5497         thread_ip_inc(p);
5498 }
5499
5500 static inline void
5501 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
5502 {
5503         struct thread *t = &p->threads[p->thread_id];
5504         struct instruction *ip = t->ip;
5505         uint64_t *regarray, idx, src;
5506
5507         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
5508
5509         /* Structs. */
5510         regarray = instr_regarray_regarray(p, ip);
5511         idx = instr_regarray_idx_hbo(p, t, ip);
5512         src = ip->regarray.dstsrc_val;
5513         regarray[idx] = src;
5514
5515         /* Thread. */
5516         thread_ip_inc(p);
5517 }
5518
5519 static inline void
5520 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
5521 {
5522         struct thread *t = &p->threads[p->thread_id];
5523         struct instruction *ip = t->ip;
5524         uint64_t *regarray, idx, src;
5525
5526         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
5527
5528         /* Structs. */
5529         regarray = instr_regarray_regarray(p, ip);
5530         idx = instr_regarray_idx_imm(p, ip);
5531         src = instr_regarray_src_nbo(t, ip);
5532         regarray[idx] = src;
5533
5534         /* Thread. */
5535         thread_ip_inc(p);
5536 }
5537
5538 static inline void
5539 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
5540 {
5541         struct thread *t = &p->threads[p->thread_id];
5542         struct instruction *ip = t->ip;
5543         uint64_t *regarray, idx, src;
5544
5545         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
5546
5547         /* Structs. */
5548         regarray = instr_regarray_regarray(p, ip);
5549         idx = instr_regarray_idx_imm(p, ip);
5550         src = instr_regarray_src_hbo(t, ip);
5551         regarray[idx] = src;
5552
5553         /* Thread. */
5554         thread_ip_inc(p);
5555 }
5556
5557 static inline void
5558 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
5559 {
5560         struct thread *t = &p->threads[p->thread_id];
5561         struct instruction *ip = t->ip;
5562         uint64_t *regarray, idx, src;
5563
5564         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
5565
5566         /* Structs. */
5567         regarray = instr_regarray_regarray(p, ip);
5568         idx = instr_regarray_idx_imm(p, ip);
5569         src = ip->regarray.dstsrc_val;
5570         regarray[idx] = src;
5571
5572         /* Thread. */
5573         thread_ip_inc(p);
5574 }
5575
5576 static inline void
5577 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
5578 {
5579         struct thread *t = &p->threads[p->thread_id];
5580         struct instruction *ip = t->ip;
5581         uint64_t *regarray, idx, src;
5582
5583         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
5584
5585         /* Structs. */
5586         regarray = instr_regarray_regarray(p, ip);
5587         idx = instr_regarray_idx_nbo(p, t, ip);
5588         src = instr_regarray_src_nbo(t, ip);
5589         regarray[idx] += src;
5590
5591         /* Thread. */
5592         thread_ip_inc(p);
5593 }
5594
5595 static inline void
5596 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
5597 {
5598         struct thread *t = &p->threads[p->thread_id];
5599         struct instruction *ip = t->ip;
5600         uint64_t *regarray, idx, src;
5601
5602         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
5603
5604         /* Structs. */
5605         regarray = instr_regarray_regarray(p, ip);
5606         idx = instr_regarray_idx_nbo(p, t, ip);
5607         src = instr_regarray_src_hbo(t, ip);
5608         regarray[idx] += src;
5609
5610         /* Thread. */
5611         thread_ip_inc(p);
5612 }
5613
5614 static inline void
5615 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
5616 {
5617         struct thread *t = &p->threads[p->thread_id];
5618         struct instruction *ip = t->ip;
5619         uint64_t *regarray, idx, src;
5620
5621         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
5622
5623         /* Structs. */
5624         regarray = instr_regarray_regarray(p, ip);
5625         idx = instr_regarray_idx_hbo(p, t, ip);
5626         src = instr_regarray_src_nbo(t, ip);
5627         regarray[idx] += src;
5628
5629         /* Thread. */
5630         thread_ip_inc(p);
5631 }
5632
5633 static inline void
5634 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
5635 {
5636         struct thread *t = &p->threads[p->thread_id];
5637         struct instruction *ip = t->ip;
5638         uint64_t *regarray, idx, src;
5639
5640         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
5641
5642         /* Structs. */
5643         regarray = instr_regarray_regarray(p, ip);
5644         idx = instr_regarray_idx_hbo(p, t, ip);
5645         src = instr_regarray_src_hbo(t, ip);
5646         regarray[idx] += src;
5647
5648         /* Thread. */
5649         thread_ip_inc(p);
5650 }
5651
5652 static inline void
5653 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
5654 {
5655         struct thread *t = &p->threads[p->thread_id];
5656         struct instruction *ip = t->ip;
5657         uint64_t *regarray, idx, src;
5658
5659         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
5660
5661         /* Structs. */
5662         regarray = instr_regarray_regarray(p, ip);
5663         idx = instr_regarray_idx_nbo(p, t, ip);
5664         src = ip->regarray.dstsrc_val;
5665         regarray[idx] += src;
5666
5667         /* Thread. */
5668         thread_ip_inc(p);
5669 }
5670
5671 static inline void
5672 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
5673 {
5674         struct thread *t = &p->threads[p->thread_id];
5675         struct instruction *ip = t->ip;
5676         uint64_t *regarray, idx, src;
5677
5678         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
5679
5680         /* Structs. */
5681         regarray = instr_regarray_regarray(p, ip);
5682         idx = instr_regarray_idx_hbo(p, t, ip);
5683         src = ip->regarray.dstsrc_val;
5684         regarray[idx] += src;
5685
5686         /* Thread. */
5687         thread_ip_inc(p);
5688 }
5689
5690 static inline void
5691 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
5692 {
5693         struct thread *t = &p->threads[p->thread_id];
5694         struct instruction *ip = t->ip;
5695         uint64_t *regarray, idx, src;
5696
5697         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
5698
5699         /* Structs. */
5700         regarray = instr_regarray_regarray(p, ip);
5701         idx = instr_regarray_idx_imm(p, ip);
5702         src = instr_regarray_src_nbo(t, ip);
5703         regarray[idx] += src;
5704
5705         /* Thread. */
5706         thread_ip_inc(p);
5707 }
5708
5709 static inline void
5710 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
5711 {
5712         struct thread *t = &p->threads[p->thread_id];
5713         struct instruction *ip = t->ip;
5714         uint64_t *regarray, idx, src;
5715
5716         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
5717
5718         /* Structs. */
5719         regarray = instr_regarray_regarray(p, ip);
5720         idx = instr_regarray_idx_imm(p, ip);
5721         src = instr_regarray_src_hbo(t, ip);
5722         regarray[idx] += src;
5723
5724         /* Thread. */
5725         thread_ip_inc(p);
5726 }
5727
5728 static inline void
5729 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
5730 {
5731         struct thread *t = &p->threads[p->thread_id];
5732         struct instruction *ip = t->ip;
5733         uint64_t *regarray, idx, src;
5734
5735         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
5736
5737         /* Structs. */
5738         regarray = instr_regarray_regarray(p, ip);
5739         idx = instr_regarray_idx_imm(p, ip);
5740         src = ip->regarray.dstsrc_val;
5741         regarray[idx] += src;
5742
5743         /* Thread. */
5744         thread_ip_inc(p);
5745 }
5746
5747 /*
5748  * metarray.
5749  */
5750 static struct metarray *
5751 metarray_find(struct rte_swx_pipeline *p, const char *name);
5752
5753 static int
5754 instr_metprefetch_translate(struct rte_swx_pipeline *p,
5755                             struct action *action,
5756                             char **tokens,
5757                             int n_tokens,
5758                             struct instruction *instr,
5759                             struct instruction_data *data __rte_unused)
5760 {
5761         char *metarray = tokens[1], *idx = tokens[2];
5762         struct metarray *m;
5763         struct field *fidx;
5764         uint32_t idx_struct_id, idx_val;
5765
5766         CHECK(n_tokens == 3, EINVAL);
5767
5768         m = metarray_find(p, metarray);
5769         CHECK(m, EINVAL);
5770
5771         /* METPREFETCH_H, METPREFETCH_M. */
5772         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5773         if (fidx) {
5774                 instr->type = INSTR_METPREFETCH_M;
5775                 if (idx[0] == 'h')
5776                         instr->type = INSTR_METPREFETCH_H;
5777
5778                 instr->meter.metarray_id = m->id;
5779                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5780                 instr->meter.idx.n_bits = fidx->n_bits;
5781                 instr->meter.idx.offset = fidx->offset / 8;
5782                 return 0;
5783         }
5784
5785         /* METPREFETCH_I. */
5786         idx_val = strtoul(idx, &idx, 0);
5787         CHECK(!idx[0], EINVAL);
5788
5789         instr->type = INSTR_METPREFETCH_I;
5790         instr->meter.metarray_id = m->id;
5791         instr->meter.idx_val = idx_val;
5792         return 0;
5793 }
5794
5795 static int
5796 instr_meter_translate(struct rte_swx_pipeline *p,
5797                       struct action *action,
5798                       char **tokens,
5799                       int n_tokens,
5800                       struct instruction *instr,
5801                       struct instruction_data *data __rte_unused)
5802 {
5803         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
5804         char *color_in = tokens[4], *color_out = tokens[5];
5805         struct metarray *m;
5806         struct field *fidx, *flength, *fcin, *fcout;
5807         uint32_t idx_struct_id, length_struct_id;
5808         uint32_t color_in_struct_id, color_out_struct_id;
5809
5810         CHECK(n_tokens == 6, EINVAL);
5811
5812         m = metarray_find(p, metarray);
5813         CHECK(m, EINVAL);
5814
5815         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5816
5817         flength = struct_field_parse(p, action, length, &length_struct_id);
5818         CHECK(flength, EINVAL);
5819
5820         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
5821
5822         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
5823         CHECK(fcout, EINVAL);
5824
5825         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
5826         if (fidx && fcin) {
5827                 instr->type = INSTR_METER_MMM;
5828                 if (idx[0] == 'h' && length[0] == 'h')
5829                         instr->type = INSTR_METER_HHM;
5830                 if (idx[0] == 'h' && length[0] != 'h')
5831                         instr->type = INSTR_METER_HMM;
5832                 if (idx[0] != 'h' && length[0] == 'h')
5833                         instr->type = INSTR_METER_MHM;
5834
5835                 instr->meter.metarray_id = m->id;
5836
5837                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5838                 instr->meter.idx.n_bits = fidx->n_bits;
5839                 instr->meter.idx.offset = fidx->offset / 8;
5840
5841                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5842                 instr->meter.length.n_bits = flength->n_bits;
5843                 instr->meter.length.offset = flength->offset / 8;
5844
5845                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5846                 instr->meter.color_in.n_bits = fcin->n_bits;
5847                 instr->meter.color_in.offset = fcin->offset / 8;
5848
5849                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5850                 instr->meter.color_out.n_bits = fcout->n_bits;
5851                 instr->meter.color_out.offset = fcout->offset / 8;
5852
5853                 return 0;
5854         }
5855
5856         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
5857         if (fidx && !fcin) {
5858                 uint32_t color_in_val = strtoul(color_in, &color_in, 0);
5859                 CHECK(!color_in[0], EINVAL);
5860
5861                 instr->type = INSTR_METER_MMI;
5862                 if (idx[0] == 'h' && length[0] == 'h')
5863                         instr->type = INSTR_METER_HHI;
5864                 if (idx[0] == 'h' && length[0] != 'h')
5865                         instr->type = INSTR_METER_HMI;
5866                 if (idx[0] != 'h' && length[0] == 'h')
5867                         instr->type = INSTR_METER_MHI;
5868
5869                 instr->meter.metarray_id = m->id;
5870
5871                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5872                 instr->meter.idx.n_bits = fidx->n_bits;
5873                 instr->meter.idx.offset = fidx->offset / 8;
5874
5875                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5876                 instr->meter.length.n_bits = flength->n_bits;
5877                 instr->meter.length.offset = flength->offset / 8;
5878
5879                 instr->meter.color_in_val = color_in_val;
5880
5881                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5882                 instr->meter.color_out.n_bits = fcout->n_bits;
5883                 instr->meter.color_out.offset = fcout->offset / 8;
5884
5885                 return 0;
5886         }
5887
5888         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5889         if (!fidx && fcin) {
5890                 uint32_t idx_val;
5891
5892                 idx_val = strtoul(idx, &idx, 0);
5893                 CHECK(!idx[0], EINVAL);
5894
5895                 instr->type = INSTR_METER_IMM;
5896                 if (length[0] == 'h')
5897                         instr->type = INSTR_METER_IHM;
5898
5899                 instr->meter.metarray_id = m->id;
5900
5901                 instr->meter.idx_val = idx_val;
5902
5903                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5904                 instr->meter.length.n_bits = flength->n_bits;
5905                 instr->meter.length.offset = flength->offset / 8;
5906
5907                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5908                 instr->meter.color_in.n_bits = fcin->n_bits;
5909                 instr->meter.color_in.offset = fcin->offset / 8;
5910
5911                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5912                 instr->meter.color_out.n_bits = fcout->n_bits;
5913                 instr->meter.color_out.offset = fcout->offset / 8;
5914
5915                 return 0;
5916         }
5917
5918         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5919         if (!fidx && !fcin) {
5920                 uint32_t idx_val, color_in_val;
5921
5922                 idx_val = strtoul(idx, &idx, 0);
5923                 CHECK(!idx[0], EINVAL);
5924
5925                 color_in_val = strtoul(color_in, &color_in, 0);
5926                 CHECK(!color_in[0], EINVAL);
5927
5928                 instr->type = INSTR_METER_IMI;
5929                 if (length[0] == 'h')
5930                         instr->type = INSTR_METER_IHI;
5931
5932                 instr->meter.metarray_id = m->id;
5933
5934                 instr->meter.idx_val = idx_val;
5935
5936                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5937                 instr->meter.length.n_bits = flength->n_bits;
5938                 instr->meter.length.offset = flength->offset / 8;
5939
5940                 instr->meter.color_in_val = color_in_val;
5941
5942                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5943                 instr->meter.color_out.n_bits = fcout->n_bits;
5944                 instr->meter.color_out.offset = fcout->offset / 8;
5945
5946                 return 0;
5947         }
5948
5949         CHECK(0, EINVAL);
5950 }
5951
5952 static inline struct meter *
5953 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5954 {
5955         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5956
5957         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5958         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5959         uint64_t idx64 = *idx64_ptr;
5960         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
5961         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5962
5963         return &r->metarray[idx];
5964 }
5965
5966 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5967
5968 static inline struct meter *
5969 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5970 {
5971         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5972
5973         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5974         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5975         uint64_t idx64 = *idx64_ptr;
5976         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
5977
5978         return &r->metarray[idx];
5979 }
5980
5981 #else
5982
5983 #define instr_meter_idx_nbo instr_meter_idx_hbo
5984
5985 #endif
5986
5987 static inline struct meter *
5988 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5989 {
5990         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5991
5992         uint64_t idx =  ip->meter.idx_val & r->size_mask;
5993
5994         return &r->metarray[idx];
5995 }
5996
5997 static inline uint32_t
5998 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
5999 {
6000         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6001         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6002         uint64_t src64 = *src64_ptr;
6003         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
6004         uint64_t src = src64 & src64_mask;
6005
6006         return (uint32_t)src;
6007 }
6008
6009 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6010
6011 static inline uint32_t
6012 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
6013 {
6014         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6015         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6016         uint64_t src64 = *src64_ptr;
6017         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
6018
6019         return (uint32_t)src;
6020 }
6021
6022 #else
6023
6024 #define instr_meter_length_nbo instr_meter_length_hbo
6025
6026 #endif
6027
6028 static inline enum rte_color
6029 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
6030 {
6031         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
6032         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
6033         uint64_t src64 = *src64_ptr;
6034         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
6035         uint64_t src = src64 & src64_mask;
6036
6037         return (enum rte_color)src;
6038 }
6039
6040 static inline void
6041 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
6042 {
6043         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
6044         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
6045         uint64_t dst64 = *dst64_ptr;
6046         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
6047
6048         uint64_t src = (uint64_t)color_out;
6049
6050         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6051 }
6052
6053 static inline void
6054 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
6055 {
6056         struct thread *t = &p->threads[p->thread_id];
6057         struct instruction *ip = t->ip;
6058         struct meter *m;
6059
6060         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
6061
6062         /* Structs. */
6063         m = instr_meter_idx_nbo(p, t, ip);
6064         rte_prefetch0(m);
6065
6066         /* Thread. */
6067         thread_ip_inc(p);
6068 }
6069
6070 static inline void
6071 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
6072 {
6073         struct thread *t = &p->threads[p->thread_id];
6074         struct instruction *ip = t->ip;
6075         struct meter *m;
6076
6077         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
6078
6079         /* Structs. */
6080         m = instr_meter_idx_hbo(p, t, ip);
6081         rte_prefetch0(m);
6082
6083         /* Thread. */
6084         thread_ip_inc(p);
6085 }
6086
6087 static inline void
6088 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
6089 {
6090         struct thread *t = &p->threads[p->thread_id];
6091         struct instruction *ip = t->ip;
6092         struct meter *m;
6093
6094         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
6095
6096         /* Structs. */
6097         m = instr_meter_idx_imm(p, ip);
6098         rte_prefetch0(m);
6099
6100         /* Thread. */
6101         thread_ip_inc(p);
6102 }
6103
6104 static inline void
6105 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
6106 {
6107         struct thread *t = &p->threads[p->thread_id];
6108         struct instruction *ip = t->ip;
6109         struct meter *m;
6110         uint64_t time, n_pkts, n_bytes;
6111         uint32_t length;
6112         enum rte_color color_in, color_out;
6113
6114         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
6115
6116         /* Structs. */
6117         m = instr_meter_idx_nbo(p, t, ip);
6118         rte_prefetch0(m->n_pkts);
6119         time = rte_get_tsc_cycles();
6120         length = instr_meter_length_nbo(t, ip);
6121         color_in = instr_meter_color_in_hbo(t, ip);
6122
6123         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6124                 &m->profile->profile,
6125                 time,
6126                 length,
6127                 color_in);
6128
6129         color_out &= m->color_mask;
6130
6131         n_pkts = m->n_pkts[color_out];
6132         n_bytes = m->n_bytes[color_out];
6133
6134         instr_meter_color_out_hbo_set(t, ip, color_out);
6135
6136         m->n_pkts[color_out] = n_pkts + 1;
6137         m->n_bytes[color_out] = n_bytes + length;
6138
6139         /* Thread. */
6140         thread_ip_inc(p);
6141 }
6142
6143 static inline void
6144 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
6145 {
6146         struct thread *t = &p->threads[p->thread_id];
6147         struct instruction *ip = t->ip;
6148         struct meter *m;
6149         uint64_t time, n_pkts, n_bytes;
6150         uint32_t length;
6151         enum rte_color color_in, color_out;
6152
6153         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
6154
6155         /* Structs. */
6156         m = instr_meter_idx_nbo(p, t, ip);
6157         rte_prefetch0(m->n_pkts);
6158         time = rte_get_tsc_cycles();
6159         length = instr_meter_length_nbo(t, ip);
6160         color_in = (enum rte_color)ip->meter.color_in_val;
6161
6162         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6163                 &m->profile->profile,
6164                 time,
6165                 length,
6166                 color_in);
6167
6168         color_out &= m->color_mask;
6169
6170         n_pkts = m->n_pkts[color_out];
6171         n_bytes = m->n_bytes[color_out];
6172
6173         instr_meter_color_out_hbo_set(t, ip, color_out);
6174
6175         m->n_pkts[color_out] = n_pkts + 1;
6176         m->n_bytes[color_out] = n_bytes + length;
6177
6178         /* Thread. */
6179         thread_ip_inc(p);
6180 }
6181
6182 static inline void
6183 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
6184 {
6185         struct thread *t = &p->threads[p->thread_id];
6186         struct instruction *ip = t->ip;
6187         struct meter *m;
6188         uint64_t time, n_pkts, n_bytes;
6189         uint32_t length;
6190         enum rte_color color_in, color_out;
6191
6192         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
6193
6194         /* Structs. */
6195         m = instr_meter_idx_nbo(p, t, ip);
6196         rte_prefetch0(m->n_pkts);
6197         time = rte_get_tsc_cycles();
6198         length = instr_meter_length_hbo(t, ip);
6199         color_in = instr_meter_color_in_hbo(t, ip);
6200
6201         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6202                 &m->profile->profile,
6203                 time,
6204                 length,
6205                 color_in);
6206
6207         color_out &= m->color_mask;
6208
6209         n_pkts = m->n_pkts[color_out];
6210         n_bytes = m->n_bytes[color_out];
6211
6212         instr_meter_color_out_hbo_set(t, ip, color_out);
6213
6214         m->n_pkts[color_out] = n_pkts + 1;
6215         m->n_bytes[color_out] = n_bytes + length;
6216
6217         /* Thread. */
6218         thread_ip_inc(p);
6219 }
6220 static inline void
6221 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
6222 {
6223         struct thread *t = &p->threads[p->thread_id];
6224         struct instruction *ip = t->ip;
6225         struct meter *m;
6226         uint64_t time, n_pkts, n_bytes;
6227         uint32_t length;
6228         enum rte_color color_in, color_out;
6229
6230         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
6231
6232         /* Structs. */
6233         m = instr_meter_idx_nbo(p, t, ip);
6234         rte_prefetch0(m->n_pkts);
6235         time = rte_get_tsc_cycles();
6236         length = instr_meter_length_hbo(t, ip);
6237         color_in = (enum rte_color)ip->meter.color_in_val;
6238
6239         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6240                 &m->profile->profile,
6241                 time,
6242                 length,
6243                 color_in);
6244
6245         color_out &= m->color_mask;
6246
6247         n_pkts = m->n_pkts[color_out];
6248         n_bytes = m->n_bytes[color_out];
6249
6250         instr_meter_color_out_hbo_set(t, ip, color_out);
6251
6252         m->n_pkts[color_out] = n_pkts + 1;
6253         m->n_bytes[color_out] = n_bytes + length;
6254
6255         /* Thread. */
6256         thread_ip_inc(p);
6257 }
6258
6259 static inline void
6260 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
6261 {
6262         struct thread *t = &p->threads[p->thread_id];
6263         struct instruction *ip = t->ip;
6264         struct meter *m;
6265         uint64_t time, n_pkts, n_bytes;
6266         uint32_t length;
6267         enum rte_color color_in, color_out;
6268
6269         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
6270
6271         /* Structs. */
6272         m = instr_meter_idx_hbo(p, t, ip);
6273         rte_prefetch0(m->n_pkts);
6274         time = rte_get_tsc_cycles();
6275         length = instr_meter_length_nbo(t, ip);
6276         color_in = instr_meter_color_in_hbo(t, ip);
6277
6278         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6279                 &m->profile->profile,
6280                 time,
6281                 length,
6282                 color_in);
6283
6284         color_out &= m->color_mask;
6285
6286         n_pkts = m->n_pkts[color_out];
6287         n_bytes = m->n_bytes[color_out];
6288
6289         instr_meter_color_out_hbo_set(t, ip, color_out);
6290
6291         m->n_pkts[color_out] = n_pkts + 1;
6292         m->n_bytes[color_out] = n_bytes + length;
6293
6294         /* Thread. */
6295         thread_ip_inc(p);
6296 }
6297
6298 static inline void
6299 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
6300 {
6301         struct thread *t = &p->threads[p->thread_id];
6302         struct instruction *ip = t->ip;
6303         struct meter *m;
6304         uint64_t time, n_pkts, n_bytes;
6305         uint32_t length;
6306         enum rte_color color_in, color_out;
6307
6308         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
6309
6310         /* Structs. */
6311         m = instr_meter_idx_hbo(p, t, ip);
6312         rte_prefetch0(m->n_pkts);
6313         time = rte_get_tsc_cycles();
6314         length = instr_meter_length_nbo(t, ip);
6315         color_in = (enum rte_color)ip->meter.color_in_val;
6316
6317         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6318                 &m->profile->profile,
6319                 time,
6320                 length,
6321                 color_in);
6322
6323         color_out &= m->color_mask;
6324
6325         n_pkts = m->n_pkts[color_out];
6326         n_bytes = m->n_bytes[color_out];
6327
6328         instr_meter_color_out_hbo_set(t, ip, color_out);
6329
6330         m->n_pkts[color_out] = n_pkts + 1;
6331         m->n_bytes[color_out] = n_bytes + length;
6332
6333         /* Thread. */
6334         thread_ip_inc(p);
6335 }
6336
6337 static inline void
6338 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
6339 {
6340         struct thread *t = &p->threads[p->thread_id];
6341         struct instruction *ip = t->ip;
6342         struct meter *m;
6343         uint64_t time, n_pkts, n_bytes;
6344         uint32_t length;
6345         enum rte_color color_in, color_out;
6346
6347         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
6348
6349         /* Structs. */
6350         m = instr_meter_idx_hbo(p, t, ip);
6351         rte_prefetch0(m->n_pkts);
6352         time = rte_get_tsc_cycles();
6353         length = instr_meter_length_hbo(t, ip);
6354         color_in = instr_meter_color_in_hbo(t, ip);
6355
6356         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6357                 &m->profile->profile,
6358                 time,
6359                 length,
6360                 color_in);
6361
6362         color_out &= m->color_mask;
6363
6364         n_pkts = m->n_pkts[color_out];
6365         n_bytes = m->n_bytes[color_out];
6366
6367         instr_meter_color_out_hbo_set(t, ip, color_out);
6368
6369         m->n_pkts[color_out] = n_pkts + 1;
6370         m->n_bytes[color_out] = n_bytes + length;
6371
6372         /* Thread. */
6373         thread_ip_inc(p);
6374 }
6375
6376 static inline void
6377 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
6378 {
6379         struct thread *t = &p->threads[p->thread_id];
6380         struct instruction *ip = t->ip;
6381         struct meter *m;
6382         uint64_t time, n_pkts, n_bytes;
6383         uint32_t length;
6384         enum rte_color color_in, color_out;
6385
6386         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
6387
6388         /* Structs. */
6389         m = instr_meter_idx_hbo(p, t, ip);
6390         rte_prefetch0(m->n_pkts);
6391         time = rte_get_tsc_cycles();
6392         length = instr_meter_length_hbo(t, ip);
6393         color_in = (enum rte_color)ip->meter.color_in_val;
6394
6395         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6396                 &m->profile->profile,
6397                 time,
6398                 length,
6399                 color_in);
6400
6401         color_out &= m->color_mask;
6402
6403         n_pkts = m->n_pkts[color_out];
6404         n_bytes = m->n_bytes[color_out];
6405
6406         instr_meter_color_out_hbo_set(t, ip, color_out);
6407
6408         m->n_pkts[color_out] = n_pkts + 1;
6409         m->n_bytes[color_out] = n_bytes + length;
6410
6411         /* Thread. */
6412         thread_ip_inc(p);
6413 }
6414
6415 static inline void
6416 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
6417 {
6418         struct thread *t = &p->threads[p->thread_id];
6419         struct instruction *ip = t->ip;
6420         struct meter *m;
6421         uint64_t time, n_pkts, n_bytes;
6422         uint32_t length;
6423         enum rte_color color_in, color_out;
6424
6425         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
6426
6427         /* Structs. */
6428         m = instr_meter_idx_imm(p, ip);
6429         rte_prefetch0(m->n_pkts);
6430         time = rte_get_tsc_cycles();
6431         length = instr_meter_length_nbo(t, ip);
6432         color_in = instr_meter_color_in_hbo(t, ip);
6433
6434         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6435                 &m->profile->profile,
6436                 time,
6437                 length,
6438                 color_in);
6439
6440         color_out &= m->color_mask;
6441
6442         n_pkts = m->n_pkts[color_out];
6443         n_bytes = m->n_bytes[color_out];
6444
6445         instr_meter_color_out_hbo_set(t, ip, color_out);
6446
6447         m->n_pkts[color_out] = n_pkts + 1;
6448         m->n_bytes[color_out] = n_bytes + length;
6449
6450         /* Thread. */
6451         thread_ip_inc(p);
6452 }
6453
6454 static inline void
6455 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
6456 {
6457         struct thread *t = &p->threads[p->thread_id];
6458         struct instruction *ip = t->ip;
6459         struct meter *m;
6460         uint64_t time, n_pkts, n_bytes;
6461         uint32_t length;
6462         enum rte_color color_in, color_out;
6463
6464         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
6465
6466         /* Structs. */
6467         m = instr_meter_idx_imm(p, ip);
6468         rte_prefetch0(m->n_pkts);
6469         time = rte_get_tsc_cycles();
6470         length = instr_meter_length_nbo(t, ip);
6471         color_in = (enum rte_color)ip->meter.color_in_val;
6472
6473         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6474                 &m->profile->profile,
6475                 time,
6476                 length,
6477                 color_in);
6478
6479         color_out &= m->color_mask;
6480
6481         n_pkts = m->n_pkts[color_out];
6482         n_bytes = m->n_bytes[color_out];
6483
6484         instr_meter_color_out_hbo_set(t, ip, color_out);
6485
6486         m->n_pkts[color_out] = n_pkts + 1;
6487         m->n_bytes[color_out] = n_bytes + length;
6488
6489         /* Thread. */
6490         thread_ip_inc(p);
6491 }
6492
6493 static inline void
6494 instr_meter_imm_exec(struct rte_swx_pipeline *p)
6495 {
6496         struct thread *t = &p->threads[p->thread_id];
6497         struct instruction *ip = t->ip;
6498         struct meter *m;
6499         uint64_t time, n_pkts, n_bytes;
6500         uint32_t length;
6501         enum rte_color color_in, color_out;
6502
6503         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
6504
6505         /* Structs. */
6506         m = instr_meter_idx_imm(p, ip);
6507         rte_prefetch0(m->n_pkts);
6508         time = rte_get_tsc_cycles();
6509         length = instr_meter_length_hbo(t, ip);
6510         color_in = instr_meter_color_in_hbo(t, ip);
6511
6512         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6513                 &m->profile->profile,
6514                 time,
6515                 length,
6516                 color_in);
6517
6518         color_out &= m->color_mask;
6519
6520         n_pkts = m->n_pkts[color_out];
6521         n_bytes = m->n_bytes[color_out];
6522
6523         instr_meter_color_out_hbo_set(t, ip, color_out);
6524
6525         m->n_pkts[color_out] = n_pkts + 1;
6526         m->n_bytes[color_out] = n_bytes + length;
6527
6528         /* Thread. */
6529         thread_ip_inc(p);
6530 }
6531 static inline void
6532 instr_meter_imi_exec(struct rte_swx_pipeline *p)
6533 {
6534         struct thread *t = &p->threads[p->thread_id];
6535         struct instruction *ip = t->ip;
6536         struct meter *m;
6537         uint64_t time, n_pkts, n_bytes;
6538         uint32_t length;
6539         enum rte_color color_in, color_out;
6540
6541         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
6542
6543         /* Structs. */
6544         m = instr_meter_idx_imm(p, ip);
6545         rte_prefetch0(m->n_pkts);
6546         time = rte_get_tsc_cycles();
6547         length = instr_meter_length_hbo(t, ip);
6548         color_in = (enum rte_color)ip->meter.color_in_val;
6549
6550         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6551                 &m->profile->profile,
6552                 time,
6553                 length,
6554                 color_in);
6555
6556         color_out &= m->color_mask;
6557
6558         n_pkts = m->n_pkts[color_out];
6559         n_bytes = m->n_bytes[color_out];
6560
6561         instr_meter_color_out_hbo_set(t, ip, color_out);
6562
6563         m->n_pkts[color_out] = n_pkts + 1;
6564         m->n_bytes[color_out] = n_bytes + length;
6565
6566         /* Thread. */
6567         thread_ip_inc(p);
6568 }
6569
6570 /*
6571  * jmp.
6572  */
6573 static struct action *
6574 action_find(struct rte_swx_pipeline *p, const char *name);
6575
6576 static int
6577 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
6578                     struct action *action __rte_unused,
6579                     char **tokens,
6580                     int n_tokens,
6581                     struct instruction *instr,
6582                     struct instruction_data *data)
6583 {
6584         CHECK(n_tokens == 2, EINVAL);
6585
6586         strcpy(data->jmp_label, tokens[1]);
6587
6588         instr->type = INSTR_JMP;
6589         instr->jmp.ip = NULL; /* Resolved later. */
6590         return 0;
6591 }
6592
6593 static int
6594 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
6595                           struct action *action __rte_unused,
6596                           char **tokens,
6597                           int n_tokens,
6598                           struct instruction *instr,
6599                           struct instruction_data *data)
6600 {
6601         struct header *h;
6602
6603         CHECK(n_tokens == 3, EINVAL);
6604
6605         strcpy(data->jmp_label, tokens[1]);
6606
6607         h = header_parse(p, tokens[2]);
6608         CHECK(h, EINVAL);
6609
6610         instr->type = INSTR_JMP_VALID;
6611         instr->jmp.ip = NULL; /* Resolved later. */
6612         instr->jmp.header_id = h->id;
6613         return 0;
6614 }
6615
6616 static int
6617 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
6618                             struct action *action __rte_unused,
6619                             char **tokens,
6620                             int n_tokens,
6621                             struct instruction *instr,
6622                             struct instruction_data *data)
6623 {
6624         struct header *h;
6625
6626         CHECK(n_tokens == 3, EINVAL);
6627
6628         strcpy(data->jmp_label, tokens[1]);
6629
6630         h = header_parse(p, tokens[2]);
6631         CHECK(h, EINVAL);
6632
6633         instr->type = INSTR_JMP_INVALID;
6634         instr->jmp.ip = NULL; /* Resolved later. */
6635         instr->jmp.header_id = h->id;
6636         return 0;
6637 }
6638
6639 static int
6640 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
6641                         struct action *action,
6642                         char **tokens,
6643                         int n_tokens,
6644                         struct instruction *instr,
6645                         struct instruction_data *data)
6646 {
6647         CHECK(!action, EINVAL);
6648         CHECK(n_tokens == 2, EINVAL);
6649
6650         strcpy(data->jmp_label, tokens[1]);
6651
6652         instr->type = INSTR_JMP_HIT;
6653         instr->jmp.ip = NULL; /* Resolved later. */
6654         return 0;
6655 }
6656
6657 static int
6658 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
6659                          struct action *action,
6660                          char **tokens,
6661                          int n_tokens,
6662                          struct instruction *instr,
6663                          struct instruction_data *data)
6664 {
6665         CHECK(!action, EINVAL);
6666         CHECK(n_tokens == 2, EINVAL);
6667
6668         strcpy(data->jmp_label, tokens[1]);
6669
6670         instr->type = INSTR_JMP_MISS;
6671         instr->jmp.ip = NULL; /* Resolved later. */
6672         return 0;
6673 }
6674
6675 static int
6676 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
6677                                struct action *action,
6678                                char **tokens,
6679                                int n_tokens,
6680                                struct instruction *instr,
6681                                struct instruction_data *data)
6682 {
6683         struct action *a;
6684
6685         CHECK(!action, EINVAL);
6686         CHECK(n_tokens == 3, EINVAL);
6687
6688         strcpy(data->jmp_label, tokens[1]);
6689
6690         a = action_find(p, tokens[2]);
6691         CHECK(a, EINVAL);
6692
6693         instr->type = INSTR_JMP_ACTION_HIT;
6694         instr->jmp.ip = NULL; /* Resolved later. */
6695         instr->jmp.action_id = a->id;
6696         return 0;
6697 }
6698
6699 static int
6700 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
6701                                 struct action *action,
6702                                 char **tokens,
6703                                 int n_tokens,
6704                                 struct instruction *instr,
6705                                 struct instruction_data *data)
6706 {
6707         struct action *a;
6708
6709         CHECK(!action, EINVAL);
6710         CHECK(n_tokens == 3, EINVAL);
6711
6712         strcpy(data->jmp_label, tokens[1]);
6713
6714         a = action_find(p, tokens[2]);
6715         CHECK(a, EINVAL);
6716
6717         instr->type = INSTR_JMP_ACTION_MISS;
6718         instr->jmp.ip = NULL; /* Resolved later. */
6719         instr->jmp.action_id = a->id;
6720         return 0;
6721 }
6722
6723 static int
6724 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
6725                        struct action *action,
6726                        char **tokens,
6727                        int n_tokens,
6728                        struct instruction *instr,
6729                        struct instruction_data *data)
6730 {
6731         char *a = tokens[2], *b = tokens[3];
6732         struct field *fa, *fb;
6733         uint64_t b_val;
6734         uint32_t a_struct_id, b_struct_id;
6735
6736         CHECK(n_tokens == 4, EINVAL);
6737
6738         strcpy(data->jmp_label, tokens[1]);
6739
6740         fa = struct_field_parse(p, action, a, &a_struct_id);
6741         CHECK(fa, EINVAL);
6742
6743         /* JMP_EQ or JMP_EQ_S. */
6744         fb = struct_field_parse(p, action, b, &b_struct_id);
6745         if (fb) {
6746                 instr->type = INSTR_JMP_EQ;
6747                 if ((a[0] == 'h' && b[0] != 'h') ||
6748                     (a[0] != 'h' && b[0] == 'h'))
6749                         instr->type = INSTR_JMP_EQ_S;
6750                 instr->jmp.ip = NULL; /* Resolved later. */
6751
6752                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6753                 instr->jmp.a.n_bits = fa->n_bits;
6754                 instr->jmp.a.offset = fa->offset / 8;
6755                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6756                 instr->jmp.b.n_bits = fb->n_bits;
6757                 instr->jmp.b.offset = fb->offset / 8;
6758                 return 0;
6759         }
6760
6761         /* JMP_EQ_I. */
6762         b_val = strtoull(b, &b, 0);
6763         CHECK(!b[0], EINVAL);
6764
6765         if (a[0] == 'h')
6766                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6767
6768         instr->type = INSTR_JMP_EQ_I;
6769         instr->jmp.ip = NULL; /* Resolved later. */
6770         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6771         instr->jmp.a.n_bits = fa->n_bits;
6772         instr->jmp.a.offset = fa->offset / 8;
6773         instr->jmp.b_val = b_val;
6774         return 0;
6775 }
6776
6777 static int
6778 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
6779                         struct action *action,
6780                         char **tokens,
6781                         int n_tokens,
6782                         struct instruction *instr,
6783                         struct instruction_data *data)
6784 {
6785         char *a = tokens[2], *b = tokens[3];
6786         struct field *fa, *fb;
6787         uint64_t b_val;
6788         uint32_t a_struct_id, b_struct_id;
6789
6790         CHECK(n_tokens == 4, EINVAL);
6791
6792         strcpy(data->jmp_label, tokens[1]);
6793
6794         fa = struct_field_parse(p, action, a, &a_struct_id);
6795         CHECK(fa, EINVAL);
6796
6797         /* JMP_NEQ or JMP_NEQ_S. */
6798         fb = struct_field_parse(p, action, b, &b_struct_id);
6799         if (fb) {
6800                 instr->type = INSTR_JMP_NEQ;
6801                 if ((a[0] == 'h' && b[0] != 'h') ||
6802                     (a[0] != 'h' && b[0] == 'h'))
6803                         instr->type = INSTR_JMP_NEQ_S;
6804                 instr->jmp.ip = NULL; /* Resolved later. */
6805
6806                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6807                 instr->jmp.a.n_bits = fa->n_bits;
6808                 instr->jmp.a.offset = fa->offset / 8;
6809                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6810                 instr->jmp.b.n_bits = fb->n_bits;
6811                 instr->jmp.b.offset = fb->offset / 8;
6812                 return 0;
6813         }
6814
6815         /* JMP_NEQ_I. */
6816         b_val = strtoull(b, &b, 0);
6817         CHECK(!b[0], EINVAL);
6818
6819         if (a[0] == 'h')
6820                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6821
6822         instr->type = INSTR_JMP_NEQ_I;
6823         instr->jmp.ip = NULL; /* Resolved later. */
6824         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6825         instr->jmp.a.n_bits = fa->n_bits;
6826         instr->jmp.a.offset = fa->offset / 8;
6827         instr->jmp.b_val = b_val;
6828         return 0;
6829 }
6830
6831 static int
6832 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
6833                        struct action *action,
6834                        char **tokens,
6835                        int n_tokens,
6836                        struct instruction *instr,
6837                        struct instruction_data *data)
6838 {
6839         char *a = tokens[2], *b = tokens[3];
6840         struct field *fa, *fb;
6841         uint64_t b_val;
6842         uint32_t a_struct_id, b_struct_id;
6843
6844         CHECK(n_tokens == 4, EINVAL);
6845
6846         strcpy(data->jmp_label, tokens[1]);
6847
6848         fa = struct_field_parse(p, action, a, &a_struct_id);
6849         CHECK(fa, EINVAL);
6850
6851         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
6852         fb = struct_field_parse(p, action, b, &b_struct_id);
6853         if (fb) {
6854                 instr->type = INSTR_JMP_LT;
6855                 if (a[0] == 'h' && b[0] != 'h')
6856                         instr->type = INSTR_JMP_LT_HM;
6857                 if (a[0] != 'h' && b[0] == 'h')
6858                         instr->type = INSTR_JMP_LT_MH;
6859                 if (a[0] == 'h' && b[0] == 'h')
6860                         instr->type = INSTR_JMP_LT_HH;
6861                 instr->jmp.ip = NULL; /* Resolved later. */
6862
6863                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6864                 instr->jmp.a.n_bits = fa->n_bits;
6865                 instr->jmp.a.offset = fa->offset / 8;
6866                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6867                 instr->jmp.b.n_bits = fb->n_bits;
6868                 instr->jmp.b.offset = fb->offset / 8;
6869                 return 0;
6870         }
6871
6872         /* JMP_LT_MI, JMP_LT_HI. */
6873         b_val = strtoull(b, &b, 0);
6874         CHECK(!b[0], EINVAL);
6875
6876         instr->type = INSTR_JMP_LT_MI;
6877         if (a[0] == 'h')
6878                 instr->type = INSTR_JMP_LT_HI;
6879         instr->jmp.ip = NULL; /* Resolved later. */
6880
6881         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6882         instr->jmp.a.n_bits = fa->n_bits;
6883         instr->jmp.a.offset = fa->offset / 8;
6884         instr->jmp.b_val = b_val;
6885         return 0;
6886 }
6887
6888 static int
6889 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
6890                        struct action *action,
6891                        char **tokens,
6892                        int n_tokens,
6893                        struct instruction *instr,
6894                        struct instruction_data *data)
6895 {
6896         char *a = tokens[2], *b = tokens[3];
6897         struct field *fa, *fb;
6898         uint64_t b_val;
6899         uint32_t a_struct_id, b_struct_id;
6900
6901         CHECK(n_tokens == 4, EINVAL);
6902
6903         strcpy(data->jmp_label, tokens[1]);
6904
6905         fa = struct_field_parse(p, action, a, &a_struct_id);
6906         CHECK(fa, EINVAL);
6907
6908         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
6909         fb = struct_field_parse(p, action, b, &b_struct_id);
6910         if (fb) {
6911                 instr->type = INSTR_JMP_GT;
6912                 if (a[0] == 'h' && b[0] != 'h')
6913                         instr->type = INSTR_JMP_GT_HM;
6914                 if (a[0] != 'h' && b[0] == 'h')
6915                         instr->type = INSTR_JMP_GT_MH;
6916                 if (a[0] == 'h' && b[0] == 'h')
6917                         instr->type = INSTR_JMP_GT_HH;
6918                 instr->jmp.ip = NULL; /* Resolved later. */
6919
6920                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6921                 instr->jmp.a.n_bits = fa->n_bits;
6922                 instr->jmp.a.offset = fa->offset / 8;
6923                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6924                 instr->jmp.b.n_bits = fb->n_bits;
6925                 instr->jmp.b.offset = fb->offset / 8;
6926                 return 0;
6927         }
6928
6929         /* JMP_GT_MI, JMP_GT_HI. */
6930         b_val = strtoull(b, &b, 0);
6931         CHECK(!b[0], EINVAL);
6932
6933         instr->type = INSTR_JMP_GT_MI;
6934         if (a[0] == 'h')
6935                 instr->type = INSTR_JMP_GT_HI;
6936         instr->jmp.ip = NULL; /* Resolved later. */
6937
6938         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6939         instr->jmp.a.n_bits = fa->n_bits;
6940         instr->jmp.a.offset = fa->offset / 8;
6941         instr->jmp.b_val = b_val;
6942         return 0;
6943 }
6944
6945 static inline void
6946 instr_jmp_exec(struct rte_swx_pipeline *p)
6947 {
6948         struct thread *t = &p->threads[p->thread_id];
6949         struct instruction *ip = t->ip;
6950
6951         TRACE("[Thread %2u] jmp\n", p->thread_id);
6952
6953         thread_ip_set(t, ip->jmp.ip);
6954 }
6955
6956 static inline void
6957 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
6958 {
6959         struct thread *t = &p->threads[p->thread_id];
6960         struct instruction *ip = t->ip;
6961         uint32_t header_id = ip->jmp.header_id;
6962
6963         TRACE("[Thread %2u] jmpv\n", p->thread_id);
6964
6965         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
6966 }
6967
6968 static inline void
6969 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
6970 {
6971         struct thread *t = &p->threads[p->thread_id];
6972         struct instruction *ip = t->ip;
6973         uint32_t header_id = ip->jmp.header_id;
6974
6975         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
6976
6977         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
6978 }
6979
6980 static inline void
6981 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
6982 {
6983         struct thread *t = &p->threads[p->thread_id];
6984         struct instruction *ip = t->ip;
6985         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
6986
6987         TRACE("[Thread %2u] jmph\n", p->thread_id);
6988
6989         t->ip = ip_next[t->hit];
6990 }
6991
6992 static inline void
6993 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
6994 {
6995         struct thread *t = &p->threads[p->thread_id];
6996         struct instruction *ip = t->ip;
6997         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
6998
6999         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
7000
7001         t->ip = ip_next[t->hit];
7002 }
7003
7004 static inline void
7005 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
7006 {
7007         struct thread *t = &p->threads[p->thread_id];
7008         struct instruction *ip = t->ip;
7009
7010         TRACE("[Thread %2u] jmpa\n", p->thread_id);
7011
7012         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
7013 }
7014
7015 static inline void
7016 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
7017 {
7018         struct thread *t = &p->threads[p->thread_id];
7019         struct instruction *ip = t->ip;
7020
7021         TRACE("[Thread %2u] jmpna\n", p->thread_id);
7022
7023         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
7024 }
7025
7026 static inline void
7027 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
7028 {
7029         struct thread *t = &p->threads[p->thread_id];
7030         struct instruction *ip = t->ip;
7031
7032         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
7033
7034         JMP_CMP(t, ip, ==);
7035 }
7036
7037 static inline void
7038 instr_jmp_eq_s_exec(struct rte_swx_pipeline *p)
7039 {
7040         struct thread *t = &p->threads[p->thread_id];
7041         struct instruction *ip = t->ip;
7042
7043         TRACE("[Thread %2u] jmpeq (s)\n", p->thread_id);
7044
7045         JMP_CMP_S(t, ip, ==);
7046 }
7047
7048 static inline void
7049 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
7050 {
7051         struct thread *t = &p->threads[p->thread_id];
7052         struct instruction *ip = t->ip;
7053
7054         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
7055
7056         JMP_CMP_I(t, ip, ==);
7057 }
7058
7059 static inline void
7060 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
7061 {
7062         struct thread *t = &p->threads[p->thread_id];
7063         struct instruction *ip = t->ip;
7064
7065         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
7066
7067         JMP_CMP(t, ip, !=);
7068 }
7069
7070 static inline void
7071 instr_jmp_neq_s_exec(struct rte_swx_pipeline *p)
7072 {
7073         struct thread *t = &p->threads[p->thread_id];
7074         struct instruction *ip = t->ip;
7075
7076         TRACE("[Thread %2u] jmpneq (s)\n", p->thread_id);
7077
7078         JMP_CMP_S(t, ip, !=);
7079 }
7080
7081 static inline void
7082 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
7083 {
7084         struct thread *t = &p->threads[p->thread_id];
7085         struct instruction *ip = t->ip;
7086
7087         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
7088
7089         JMP_CMP_I(t, ip, !=);
7090 }
7091
7092 static inline void
7093 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
7094 {
7095         struct thread *t = &p->threads[p->thread_id];
7096         struct instruction *ip = t->ip;
7097
7098         TRACE("[Thread %2u] jmplt\n", p->thread_id);
7099
7100         JMP_CMP(t, ip, <);
7101 }
7102
7103 static inline void
7104 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
7105 {
7106         struct thread *t = &p->threads[p->thread_id];
7107         struct instruction *ip = t->ip;
7108
7109         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
7110
7111         JMP_CMP_MH(t, ip, <);
7112 }
7113
7114 static inline void
7115 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
7116 {
7117         struct thread *t = &p->threads[p->thread_id];
7118         struct instruction *ip = t->ip;
7119
7120         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
7121
7122         JMP_CMP_HM(t, ip, <);
7123 }
7124
7125 static inline void
7126 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
7127 {
7128         struct thread *t = &p->threads[p->thread_id];
7129         struct instruction *ip = t->ip;
7130
7131         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
7132
7133         JMP_CMP_HH(t, ip, <);
7134 }
7135
7136 static inline void
7137 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
7138 {
7139         struct thread *t = &p->threads[p->thread_id];
7140         struct instruction *ip = t->ip;
7141
7142         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
7143
7144         JMP_CMP_MI(t, ip, <);
7145 }
7146
7147 static inline void
7148 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
7149 {
7150         struct thread *t = &p->threads[p->thread_id];
7151         struct instruction *ip = t->ip;
7152
7153         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
7154
7155         JMP_CMP_HI(t, ip, <);
7156 }
7157
7158 static inline void
7159 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
7160 {
7161         struct thread *t = &p->threads[p->thread_id];
7162         struct instruction *ip = t->ip;
7163
7164         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
7165
7166         JMP_CMP(t, ip, >);
7167 }
7168
7169 static inline void
7170 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
7171 {
7172         struct thread *t = &p->threads[p->thread_id];
7173         struct instruction *ip = t->ip;
7174
7175         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
7176
7177         JMP_CMP_MH(t, ip, >);
7178 }
7179
7180 static inline void
7181 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
7182 {
7183         struct thread *t = &p->threads[p->thread_id];
7184         struct instruction *ip = t->ip;
7185
7186         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
7187
7188         JMP_CMP_HM(t, ip, >);
7189 }
7190
7191 static inline void
7192 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
7193 {
7194         struct thread *t = &p->threads[p->thread_id];
7195         struct instruction *ip = t->ip;
7196
7197         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
7198
7199         JMP_CMP_HH(t, ip, >);
7200 }
7201
7202 static inline void
7203 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
7204 {
7205         struct thread *t = &p->threads[p->thread_id];
7206         struct instruction *ip = t->ip;
7207
7208         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
7209
7210         JMP_CMP_MI(t, ip, >);
7211 }
7212
7213 static inline void
7214 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
7215 {
7216         struct thread *t = &p->threads[p->thread_id];
7217         struct instruction *ip = t->ip;
7218
7219         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
7220
7221         JMP_CMP_HI(t, ip, >);
7222 }
7223
7224 /*
7225  * return.
7226  */
7227 static int
7228 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
7229                        struct action *action,
7230                        char **tokens __rte_unused,
7231                        int n_tokens,
7232                        struct instruction *instr,
7233                        struct instruction_data *data __rte_unused)
7234 {
7235         CHECK(action, EINVAL);
7236         CHECK(n_tokens == 1, EINVAL);
7237
7238         instr->type = INSTR_RETURN;
7239         return 0;
7240 }
7241
7242 static inline void
7243 instr_return_exec(struct rte_swx_pipeline *p)
7244 {
7245         struct thread *t = &p->threads[p->thread_id];
7246
7247         TRACE("[Thread %2u] return\n", p->thread_id);
7248
7249         t->ip = t->ret;
7250 }
7251
7252 static int
7253 instr_translate(struct rte_swx_pipeline *p,
7254                 struct action *action,
7255                 char *string,
7256                 struct instruction *instr,
7257                 struct instruction_data *data)
7258 {
7259         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
7260         int n_tokens = 0, tpos = 0;
7261
7262         /* Parse the instruction string into tokens. */
7263         for ( ; ; ) {
7264                 char *token;
7265
7266                 token = strtok_r(string, " \t\v", &string);
7267                 if (!token)
7268                         break;
7269
7270                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
7271                 CHECK_NAME(token, EINVAL);
7272
7273                 tokens[n_tokens] = token;
7274                 n_tokens++;
7275         }
7276
7277         CHECK(n_tokens, EINVAL);
7278
7279         /* Handle the optional instruction label. */
7280         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
7281                 strcpy(data->label, tokens[0]);
7282
7283                 tpos += 2;
7284                 CHECK(n_tokens - tpos, EINVAL);
7285         }
7286
7287         /* Identify the instruction type. */
7288         if (!strcmp(tokens[tpos], "rx"))
7289                 return instr_rx_translate(p,
7290                                           action,
7291                                           &tokens[tpos],
7292                                           n_tokens - tpos,
7293                                           instr,
7294                                           data);
7295
7296         if (!strcmp(tokens[tpos], "tx"))
7297                 return instr_tx_translate(p,
7298                                           action,
7299                                           &tokens[tpos],
7300                                           n_tokens - tpos,
7301                                           instr,
7302                                           data);
7303
7304         if (!strcmp(tokens[tpos], "extract"))
7305                 return instr_hdr_extract_translate(p,
7306                                                    action,
7307                                                    &tokens[tpos],
7308                                                    n_tokens - tpos,
7309                                                    instr,
7310                                                    data);
7311
7312         if (!strcmp(tokens[tpos], "emit"))
7313                 return instr_hdr_emit_translate(p,
7314                                                 action,
7315                                                 &tokens[tpos],
7316                                                 n_tokens - tpos,
7317                                                 instr,
7318                                                 data);
7319
7320         if (!strcmp(tokens[tpos], "validate"))
7321                 return instr_hdr_validate_translate(p,
7322                                                     action,
7323                                                     &tokens[tpos],
7324                                                     n_tokens - tpos,
7325                                                     instr,
7326                                                     data);
7327
7328         if (!strcmp(tokens[tpos], "invalidate"))
7329                 return instr_hdr_invalidate_translate(p,
7330                                                       action,
7331                                                       &tokens[tpos],
7332                                                       n_tokens - tpos,
7333                                                       instr,
7334                                                       data);
7335
7336         if (!strcmp(tokens[tpos], "mov"))
7337                 return instr_mov_translate(p,
7338                                            action,
7339                                            &tokens[tpos],
7340                                            n_tokens - tpos,
7341                                            instr,
7342                                            data);
7343
7344         if (!strcmp(tokens[tpos], "dma"))
7345                 return instr_dma_translate(p,
7346                                            action,
7347                                            &tokens[tpos],
7348                                            n_tokens - tpos,
7349                                            instr,
7350                                            data);
7351
7352         if (!strcmp(tokens[tpos], "add"))
7353                 return instr_alu_add_translate(p,
7354                                                action,
7355                                                &tokens[tpos],
7356                                                n_tokens - tpos,
7357                                                instr,
7358                                                data);
7359
7360         if (!strcmp(tokens[tpos], "sub"))
7361                 return instr_alu_sub_translate(p,
7362                                                action,
7363                                                &tokens[tpos],
7364                                                n_tokens - tpos,
7365                                                instr,
7366                                                data);
7367
7368         if (!strcmp(tokens[tpos], "ckadd"))
7369                 return instr_alu_ckadd_translate(p,
7370                                                  action,
7371                                                  &tokens[tpos],
7372                                                  n_tokens - tpos,
7373                                                  instr,
7374                                                  data);
7375
7376         if (!strcmp(tokens[tpos], "cksub"))
7377                 return instr_alu_cksub_translate(p,
7378                                                  action,
7379                                                  &tokens[tpos],
7380                                                  n_tokens - tpos,
7381                                                  instr,
7382                                                  data);
7383
7384         if (!strcmp(tokens[tpos], "and"))
7385                 return instr_alu_and_translate(p,
7386                                                action,
7387                                                &tokens[tpos],
7388                                                n_tokens - tpos,
7389                                                instr,
7390                                                data);
7391
7392         if (!strcmp(tokens[tpos], "or"))
7393                 return instr_alu_or_translate(p,
7394                                               action,
7395                                               &tokens[tpos],
7396                                               n_tokens - tpos,
7397                                               instr,
7398                                               data);
7399
7400         if (!strcmp(tokens[tpos], "xor"))
7401                 return instr_alu_xor_translate(p,
7402                                                action,
7403                                                &tokens[tpos],
7404                                                n_tokens - tpos,
7405                                                instr,
7406                                                data);
7407
7408         if (!strcmp(tokens[tpos], "shl"))
7409                 return instr_alu_shl_translate(p,
7410                                                action,
7411                                                &tokens[tpos],
7412                                                n_tokens - tpos,
7413                                                instr,
7414                                                data);
7415
7416         if (!strcmp(tokens[tpos], "shr"))
7417                 return instr_alu_shr_translate(p,
7418                                                action,
7419                                                &tokens[tpos],
7420                                                n_tokens - tpos,
7421                                                instr,
7422                                                data);
7423
7424         if (!strcmp(tokens[tpos], "regprefetch"))
7425                 return instr_regprefetch_translate(p,
7426                                                    action,
7427                                                    &tokens[tpos],
7428                                                    n_tokens - tpos,
7429                                                    instr,
7430                                                    data);
7431
7432         if (!strcmp(tokens[tpos], "regrd"))
7433                 return instr_regrd_translate(p,
7434                                              action,
7435                                              &tokens[tpos],
7436                                              n_tokens - tpos,
7437                                              instr,
7438                                              data);
7439
7440         if (!strcmp(tokens[tpos], "regwr"))
7441                 return instr_regwr_translate(p,
7442                                              action,
7443                                              &tokens[tpos],
7444                                              n_tokens - tpos,
7445                                              instr,
7446                                              data);
7447
7448         if (!strcmp(tokens[tpos], "regadd"))
7449                 return instr_regadd_translate(p,
7450                                               action,
7451                                               &tokens[tpos],
7452                                               n_tokens - tpos,
7453                                               instr,
7454                                               data);
7455
7456         if (!strcmp(tokens[tpos], "metprefetch"))
7457                 return instr_metprefetch_translate(p,
7458                                                    action,
7459                                                    &tokens[tpos],
7460                                                    n_tokens - tpos,
7461                                                    instr,
7462                                                    data);
7463
7464         if (!strcmp(tokens[tpos], "meter"))
7465                 return instr_meter_translate(p,
7466                                              action,
7467                                              &tokens[tpos],
7468                                              n_tokens - tpos,
7469                                              instr,
7470                                              data);
7471
7472         if (!strcmp(tokens[tpos], "table"))
7473                 return instr_table_translate(p,
7474                                              action,
7475                                              &tokens[tpos],
7476                                              n_tokens - tpos,
7477                                              instr,
7478                                              data);
7479
7480         if (!strcmp(tokens[tpos], "extern"))
7481                 return instr_extern_translate(p,
7482                                               action,
7483                                               &tokens[tpos],
7484                                               n_tokens - tpos,
7485                                               instr,
7486                                               data);
7487
7488         if (!strcmp(tokens[tpos], "jmp"))
7489                 return instr_jmp_translate(p,
7490                                            action,
7491                                            &tokens[tpos],
7492                                            n_tokens - tpos,
7493                                            instr,
7494                                            data);
7495
7496         if (!strcmp(tokens[tpos], "jmpv"))
7497                 return instr_jmp_valid_translate(p,
7498                                                  action,
7499                                                  &tokens[tpos],
7500                                                  n_tokens - tpos,
7501                                                  instr,
7502                                                  data);
7503
7504         if (!strcmp(tokens[tpos], "jmpnv"))
7505                 return instr_jmp_invalid_translate(p,
7506                                                    action,
7507                                                    &tokens[tpos],
7508                                                    n_tokens - tpos,
7509                                                    instr,
7510                                                    data);
7511
7512         if (!strcmp(tokens[tpos], "jmph"))
7513                 return instr_jmp_hit_translate(p,
7514                                                action,
7515                                                &tokens[tpos],
7516                                                n_tokens - tpos,
7517                                                instr,
7518                                                data);
7519
7520         if (!strcmp(tokens[tpos], "jmpnh"))
7521                 return instr_jmp_miss_translate(p,
7522                                                 action,
7523                                                 &tokens[tpos],
7524                                                 n_tokens - tpos,
7525                                                 instr,
7526                                                 data);
7527
7528         if (!strcmp(tokens[tpos], "jmpa"))
7529                 return instr_jmp_action_hit_translate(p,
7530                                                       action,
7531                                                       &tokens[tpos],
7532                                                       n_tokens - tpos,
7533                                                       instr,
7534                                                       data);
7535
7536         if (!strcmp(tokens[tpos], "jmpna"))
7537                 return instr_jmp_action_miss_translate(p,
7538                                                        action,
7539                                                        &tokens[tpos],
7540                                                        n_tokens - tpos,
7541                                                        instr,
7542                                                        data);
7543
7544         if (!strcmp(tokens[tpos], "jmpeq"))
7545                 return instr_jmp_eq_translate(p,
7546                                               action,
7547                                               &tokens[tpos],
7548                                               n_tokens - tpos,
7549                                               instr,
7550                                               data);
7551
7552         if (!strcmp(tokens[tpos], "jmpneq"))
7553                 return instr_jmp_neq_translate(p,
7554                                                action,
7555                                                &tokens[tpos],
7556                                                n_tokens - tpos,
7557                                                instr,
7558                                                data);
7559
7560         if (!strcmp(tokens[tpos], "jmplt"))
7561                 return instr_jmp_lt_translate(p,
7562                                               action,
7563                                               &tokens[tpos],
7564                                               n_tokens - tpos,
7565                                               instr,
7566                                               data);
7567
7568         if (!strcmp(tokens[tpos], "jmpgt"))
7569                 return instr_jmp_gt_translate(p,
7570                                               action,
7571                                               &tokens[tpos],
7572                                               n_tokens - tpos,
7573                                               instr,
7574                                               data);
7575
7576         if (!strcmp(tokens[tpos], "return"))
7577                 return instr_return_translate(p,
7578                                               action,
7579                                               &tokens[tpos],
7580                                               n_tokens - tpos,
7581                                               instr,
7582                                               data);
7583
7584         CHECK(0, EINVAL);
7585 }
7586
7587 static struct instruction_data *
7588 label_find(struct instruction_data *data, uint32_t n, const char *label)
7589 {
7590         uint32_t i;
7591
7592         for (i = 0; i < n; i++)
7593                 if (!strcmp(label, data[i].label))
7594                         return &data[i];
7595
7596         return NULL;
7597 }
7598
7599 static uint32_t
7600 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
7601 {
7602         uint32_t count = 0, i;
7603
7604         if (!label[0])
7605                 return 0;
7606
7607         for (i = 0; i < n; i++)
7608                 if (!strcmp(label, data[i].jmp_label))
7609                         count++;
7610
7611         return count;
7612 }
7613
7614 static int
7615 instr_label_check(struct instruction_data *instruction_data,
7616                   uint32_t n_instructions)
7617 {
7618         uint32_t i;
7619
7620         /* Check that all instruction labels are unique. */
7621         for (i = 0; i < n_instructions; i++) {
7622                 struct instruction_data *data = &instruction_data[i];
7623                 char *label = data->label;
7624                 uint32_t j;
7625
7626                 if (!label[0])
7627                         continue;
7628
7629                 for (j = i + 1; j < n_instructions; j++)
7630                         CHECK(strcmp(label, data[j].label), EINVAL);
7631         }
7632
7633         /* Get users for each instruction label. */
7634         for (i = 0; i < n_instructions; i++) {
7635                 struct instruction_data *data = &instruction_data[i];
7636                 char *label = data->label;
7637
7638                 data->n_users = label_is_used(instruction_data,
7639                                               n_instructions,
7640                                               label);
7641         }
7642
7643         return 0;
7644 }
7645
7646 static int
7647 instr_jmp_resolve(struct instruction *instructions,
7648                   struct instruction_data *instruction_data,
7649                   uint32_t n_instructions)
7650 {
7651         uint32_t i;
7652
7653         for (i = 0; i < n_instructions; i++) {
7654                 struct instruction *instr = &instructions[i];
7655                 struct instruction_data *data = &instruction_data[i];
7656                 struct instruction_data *found;
7657
7658                 if (!instruction_is_jmp(instr))
7659                         continue;
7660
7661                 found = label_find(instruction_data,
7662                                    n_instructions,
7663                                    data->jmp_label);
7664                 CHECK(found, EINVAL);
7665
7666                 instr->jmp.ip = &instructions[found - instruction_data];
7667         }
7668
7669         return 0;
7670 }
7671
7672 static int
7673 instr_verify(struct rte_swx_pipeline *p __rte_unused,
7674              struct action *a,
7675              struct instruction *instr,
7676              struct instruction_data *data __rte_unused,
7677              uint32_t n_instructions)
7678 {
7679         if (!a) {
7680                 enum instruction_type type;
7681                 uint32_t i;
7682
7683                 /* Check that the first instruction is rx. */
7684                 CHECK(instr[0].type == INSTR_RX, EINVAL);
7685
7686                 /* Check that there is at least one tx instruction. */
7687                 for (i = 0; i < n_instructions; i++) {
7688                         type = instr[i].type;
7689
7690                         if (type == INSTR_TX)
7691                                 break;
7692                 }
7693                 CHECK(i < n_instructions, EINVAL);
7694
7695                 /* Check that the last instruction is either tx or unconditional
7696                  * jump.
7697                  */
7698                 type = instr[n_instructions - 1].type;
7699                 CHECK((type == INSTR_TX) || (type == INSTR_JMP), EINVAL);
7700         }
7701
7702         if (a) {
7703                 enum instruction_type type;
7704                 uint32_t i;
7705
7706                 /* Check that there is at least one return or tx instruction. */
7707                 for (i = 0; i < n_instructions; i++) {
7708                         type = instr[i].type;
7709
7710                         if ((type == INSTR_RETURN) || (type == INSTR_TX))
7711                                 break;
7712                 }
7713                 CHECK(i < n_instructions, EINVAL);
7714         }
7715
7716         return 0;
7717 }
7718
7719 static int
7720 instr_pattern_extract_many_detect(struct instruction *instr,
7721                                   struct instruction_data *data,
7722                                   uint32_t n_instr,
7723                                   uint32_t *n_pattern_instr)
7724 {
7725         uint32_t i;
7726
7727         for (i = 0; i < n_instr; i++) {
7728                 if (data[i].invalid)
7729                         break;
7730
7731                 if (instr[i].type != INSTR_HDR_EXTRACT)
7732                         break;
7733
7734                 if (i == RTE_DIM(instr->io.hdr.header_id))
7735                         break;
7736
7737                 if (i && data[i].n_users)
7738                         break;
7739         }
7740
7741         if (i < 2)
7742                 return 0;
7743
7744         *n_pattern_instr = i;
7745         return 1;
7746 }
7747
7748 static void
7749 instr_pattern_extract_many_optimize(struct instruction *instr,
7750                                     struct instruction_data *data,
7751                                     uint32_t n_instr)
7752 {
7753         uint32_t i;
7754
7755         for (i = 1; i < n_instr; i++) {
7756                 instr[0].type++;
7757                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7758                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7759                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7760
7761                 data[i].invalid = 1;
7762         }
7763 }
7764
7765 static int
7766 instr_pattern_emit_many_tx_detect(struct instruction *instr,
7767                                   struct instruction_data *data,
7768                                   uint32_t n_instr,
7769                                   uint32_t *n_pattern_instr)
7770 {
7771         uint32_t i;
7772
7773         for (i = 0; i < n_instr; i++) {
7774                 if (data[i].invalid)
7775                         break;
7776
7777                 if (instr[i].type != INSTR_HDR_EMIT)
7778                         break;
7779
7780                 if (i == RTE_DIM(instr->io.hdr.header_id))
7781                         break;
7782
7783                 if (i && data[i].n_users)
7784                         break;
7785         }
7786
7787         if (!i)
7788                 return 0;
7789
7790         if (instr[i].type != INSTR_TX)
7791                 return 0;
7792
7793         if (data[i].n_users)
7794                 return 0;
7795
7796         i++;
7797
7798         *n_pattern_instr = i;
7799         return 1;
7800 }
7801
7802 static void
7803 instr_pattern_emit_many_tx_optimize(struct instruction *instr,
7804                                     struct instruction_data *data,
7805                                     uint32_t n_instr)
7806 {
7807         uint32_t i;
7808
7809         /* Any emit instruction in addition to the first one. */
7810         for (i = 1; i < n_instr - 1; i++) {
7811                 instr[0].type++;
7812                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7813                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7814                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7815
7816                 data[i].invalid = 1;
7817         }
7818
7819         /* The TX instruction is the last one in the pattern. */
7820         instr[0].type++;
7821         instr[0].io.io.offset = instr[i].io.io.offset;
7822         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7823         data[i].invalid = 1;
7824 }
7825
7826 static int
7827 instr_pattern_dma_many_detect(struct instruction *instr,
7828                               struct instruction_data *data,
7829                               uint32_t n_instr,
7830                               uint32_t *n_pattern_instr)
7831 {
7832         uint32_t i;
7833
7834         for (i = 0; i < n_instr; i++) {
7835                 if (data[i].invalid)
7836                         break;
7837
7838                 if (instr[i].type != INSTR_DMA_HT)
7839                         break;
7840
7841                 if (i == RTE_DIM(instr->dma.dst.header_id))
7842                         break;
7843
7844                 if (i && data[i].n_users)
7845                         break;
7846         }
7847
7848         if (i < 2)
7849                 return 0;
7850
7851         *n_pattern_instr = i;
7852         return 1;
7853 }
7854
7855 static void
7856 instr_pattern_dma_many_optimize(struct instruction *instr,
7857                                 struct instruction_data *data,
7858                                 uint32_t n_instr)
7859 {
7860         uint32_t i;
7861
7862         for (i = 1; i < n_instr; i++) {
7863                 instr[0].type++;
7864                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7865                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7866                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7867                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7868
7869                 data[i].invalid = 1;
7870         }
7871 }
7872
7873 static uint32_t
7874 instr_optimize(struct instruction *instructions,
7875                struct instruction_data *instruction_data,
7876                uint32_t n_instructions)
7877 {
7878         uint32_t i, pos = 0;
7879
7880         for (i = 0; i < n_instructions; ) {
7881                 struct instruction *instr = &instructions[i];
7882                 struct instruction_data *data = &instruction_data[i];
7883                 uint32_t n_instr = 0;
7884                 int detected;
7885
7886                 /* Extract many. */
7887                 detected = instr_pattern_extract_many_detect(instr,
7888                                                              data,
7889                                                              n_instructions - i,
7890                                                              &n_instr);
7891                 if (detected) {
7892                         instr_pattern_extract_many_optimize(instr,
7893                                                             data,
7894                                                             n_instr);
7895                         i += n_instr;
7896                         continue;
7897                 }
7898
7899                 /* Emit many + TX. */
7900                 detected = instr_pattern_emit_many_tx_detect(instr,
7901                                                              data,
7902                                                              n_instructions - i,
7903                                                              &n_instr);
7904                 if (detected) {
7905                         instr_pattern_emit_many_tx_optimize(instr,
7906                                                             data,
7907                                                             n_instr);
7908                         i += n_instr;
7909                         continue;
7910                 }
7911
7912                 /* DMA many. */
7913                 detected = instr_pattern_dma_many_detect(instr,
7914                                                          data,
7915                                                          n_instructions - i,
7916                                                          &n_instr);
7917                 if (detected) {
7918                         instr_pattern_dma_many_optimize(instr, data, n_instr);
7919                         i += n_instr;
7920                         continue;
7921                 }
7922
7923                 /* No pattern starting at the current instruction. */
7924                 i++;
7925         }
7926
7927         /* Eliminate the invalid instructions that have been optimized out. */
7928         for (i = 0; i < n_instructions; i++) {
7929                 struct instruction *instr = &instructions[i];
7930                 struct instruction_data *data = &instruction_data[i];
7931
7932                 if (data->invalid)
7933                         continue;
7934
7935                 if (i != pos) {
7936                         memcpy(&instructions[pos], instr, sizeof(*instr));
7937                         memcpy(&instruction_data[pos], data, sizeof(*data));
7938                 }
7939
7940                 pos++;
7941         }
7942
7943         return pos;
7944 }
7945
7946 static int
7947 instruction_config(struct rte_swx_pipeline *p,
7948                    struct action *a,
7949                    const char **instructions,
7950                    uint32_t n_instructions)
7951 {
7952         struct instruction *instr = NULL;
7953         struct instruction_data *data = NULL;
7954         int err = 0;
7955         uint32_t i;
7956
7957         CHECK(n_instructions, EINVAL);
7958         CHECK(instructions, EINVAL);
7959         for (i = 0; i < n_instructions; i++)
7960                 CHECK_INSTRUCTION(instructions[i], EINVAL);
7961
7962         /* Memory allocation. */
7963         instr = calloc(n_instructions, sizeof(struct instruction));
7964         if (!instr) {
7965                 err = ENOMEM;
7966                 goto error;
7967         }
7968
7969         data = calloc(n_instructions, sizeof(struct instruction_data));
7970         if (!data) {
7971                 err = ENOMEM;
7972                 goto error;
7973         }
7974
7975         for (i = 0; i < n_instructions; i++) {
7976                 char *string = strdup(instructions[i]);
7977                 if (!string) {
7978                         err = ENOMEM;
7979                         goto error;
7980                 }
7981
7982                 err = instr_translate(p, a, string, &instr[i], &data[i]);
7983                 if (err) {
7984                         free(string);
7985                         goto error;
7986                 }
7987
7988                 free(string);
7989         }
7990
7991         err = instr_label_check(data, n_instructions);
7992         if (err)
7993                 goto error;
7994
7995         err = instr_verify(p, a, instr, data, n_instructions);
7996         if (err)
7997                 goto error;
7998
7999         n_instructions = instr_optimize(instr, data, n_instructions);
8000
8001         err = instr_jmp_resolve(instr, data, n_instructions);
8002         if (err)
8003                 goto error;
8004
8005         if (a) {
8006                 a->instructions = instr;
8007                 a->n_instructions = n_instructions;
8008         } else {
8009                 p->instructions = instr;
8010                 p->n_instructions = n_instructions;
8011         }
8012
8013         free(data);
8014         return 0;
8015
8016 error:
8017         free(data);
8018         free(instr);
8019         return err;
8020 }
8021
8022 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
8023
8024 static instr_exec_t instruction_table[] = {
8025         [INSTR_RX] = instr_rx_exec,
8026         [INSTR_TX] = instr_tx_exec,
8027
8028         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
8029         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
8030         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
8031         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
8032         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
8033         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
8034         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
8035         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
8036
8037         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
8038         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
8039         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
8040         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
8041         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
8042         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
8043         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
8044         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
8045         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
8046
8047         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
8048         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
8049
8050         [INSTR_MOV] = instr_mov_exec,
8051         [INSTR_MOV_S] = instr_mov_s_exec,
8052         [INSTR_MOV_I] = instr_mov_i_exec,
8053
8054         [INSTR_DMA_HT] = instr_dma_ht_exec,
8055         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
8056         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
8057         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
8058         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
8059         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
8060         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
8061         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
8062
8063         [INSTR_ALU_ADD] = instr_alu_add_exec,
8064         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
8065         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
8066         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
8067         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
8068         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
8069
8070         [INSTR_ALU_SUB] = instr_alu_sub_exec,
8071         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
8072         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
8073         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
8074         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
8075         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
8076
8077         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
8078         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
8079         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
8080         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
8081
8082         [INSTR_ALU_AND] = instr_alu_and_exec,
8083         [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
8084         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
8085
8086         [INSTR_ALU_OR] = instr_alu_or_exec,
8087         [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
8088         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
8089
8090         [INSTR_ALU_XOR] = instr_alu_xor_exec,
8091         [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec,
8092         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
8093
8094         [INSTR_ALU_SHL] = instr_alu_shl_exec,
8095         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
8096         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
8097         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
8098         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
8099         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
8100
8101         [INSTR_ALU_SHR] = instr_alu_shr_exec,
8102         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
8103         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
8104         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
8105         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
8106         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
8107
8108         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
8109         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
8110         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
8111
8112         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
8113         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
8114         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
8115         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
8116         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
8117         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
8118
8119         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
8120         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
8121         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
8122         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
8123         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
8124         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
8125         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
8126         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
8127         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
8128
8129         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
8130         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
8131         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
8132         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
8133         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
8134         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
8135         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
8136         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
8137         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
8138
8139         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
8140         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
8141         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
8142
8143         [INSTR_METER_HHM] = instr_meter_hhm_exec,
8144         [INSTR_METER_HHI] = instr_meter_hhi_exec,
8145         [INSTR_METER_HMM] = instr_meter_hmm_exec,
8146         [INSTR_METER_HMI] = instr_meter_hmi_exec,
8147         [INSTR_METER_MHM] = instr_meter_mhm_exec,
8148         [INSTR_METER_MHI] = instr_meter_mhi_exec,
8149         [INSTR_METER_MMM] = instr_meter_mmm_exec,
8150         [INSTR_METER_MMI] = instr_meter_mmi_exec,
8151         [INSTR_METER_IHM] = instr_meter_ihm_exec,
8152         [INSTR_METER_IHI] = instr_meter_ihi_exec,
8153         [INSTR_METER_IMM] = instr_meter_imm_exec,
8154         [INSTR_METER_IMI] = instr_meter_imi_exec,
8155
8156         [INSTR_TABLE] = instr_table_exec,
8157         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
8158         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
8159
8160         [INSTR_JMP] = instr_jmp_exec,
8161         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
8162         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
8163         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
8164         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
8165         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
8166         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
8167
8168         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
8169         [INSTR_JMP_EQ_S] = instr_jmp_eq_s_exec,
8170         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
8171
8172         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
8173         [INSTR_JMP_NEQ_S] = instr_jmp_neq_s_exec,
8174         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
8175
8176         [INSTR_JMP_LT] = instr_jmp_lt_exec,
8177         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
8178         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
8179         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
8180         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
8181         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
8182
8183         [INSTR_JMP_GT] = instr_jmp_gt_exec,
8184         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
8185         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
8186         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
8187         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
8188         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
8189
8190         [INSTR_RETURN] = instr_return_exec,
8191 };
8192
8193 static inline void
8194 instr_exec(struct rte_swx_pipeline *p)
8195 {
8196         struct thread *t = &p->threads[p->thread_id];
8197         struct instruction *ip = t->ip;
8198         instr_exec_t instr = instruction_table[ip->type];
8199
8200         instr(p);
8201 }
8202
8203 /*
8204  * Action.
8205  */
8206 static struct action *
8207 action_find(struct rte_swx_pipeline *p, const char *name)
8208 {
8209         struct action *elem;
8210
8211         if (!name)
8212                 return NULL;
8213
8214         TAILQ_FOREACH(elem, &p->actions, node)
8215                 if (strcmp(elem->name, name) == 0)
8216                         return elem;
8217
8218         return NULL;
8219 }
8220
8221 static struct action *
8222 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8223 {
8224         struct action *action = NULL;
8225
8226         TAILQ_FOREACH(action, &p->actions, node)
8227                 if (action->id == id)
8228                         return action;
8229
8230         return NULL;
8231 }
8232
8233 static struct field *
8234 action_field_find(struct action *a, const char *name)
8235 {
8236         return a->st ? struct_type_field_find(a->st, name) : NULL;
8237 }
8238
8239 static struct field *
8240 action_field_parse(struct action *action, const char *name)
8241 {
8242         if (name[0] != 't' || name[1] != '.')
8243                 return NULL;
8244
8245         return action_field_find(action, &name[2]);
8246 }
8247
8248 int
8249 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
8250                                const char *name,
8251                                const char *args_struct_type_name,
8252                                const char **instructions,
8253                                uint32_t n_instructions)
8254 {
8255         struct struct_type *args_struct_type;
8256         struct action *a;
8257         int err;
8258
8259         CHECK(p, EINVAL);
8260
8261         CHECK_NAME(name, EINVAL);
8262         CHECK(!action_find(p, name), EEXIST);
8263
8264         if (args_struct_type_name) {
8265                 CHECK_NAME(args_struct_type_name, EINVAL);
8266                 args_struct_type = struct_type_find(p, args_struct_type_name);
8267                 CHECK(args_struct_type, EINVAL);
8268         } else {
8269                 args_struct_type = NULL;
8270         }
8271
8272         /* Node allocation. */
8273         a = calloc(1, sizeof(struct action));
8274         CHECK(a, ENOMEM);
8275
8276         /* Node initialization. */
8277         strcpy(a->name, name);
8278         a->st = args_struct_type;
8279         a->id = p->n_actions;
8280
8281         /* Instruction translation. */
8282         err = instruction_config(p, a, instructions, n_instructions);
8283         if (err) {
8284                 free(a);
8285                 return err;
8286         }
8287
8288         /* Node add to tailq. */
8289         TAILQ_INSERT_TAIL(&p->actions, a, node);
8290         p->n_actions++;
8291
8292         return 0;
8293 }
8294
8295 static int
8296 action_build(struct rte_swx_pipeline *p)
8297 {
8298         struct action *action;
8299
8300         p->action_instructions = calloc(p->n_actions,
8301                                         sizeof(struct instruction *));
8302         CHECK(p->action_instructions, ENOMEM);
8303
8304         TAILQ_FOREACH(action, &p->actions, node)
8305                 p->action_instructions[action->id] = action->instructions;
8306
8307         return 0;
8308 }
8309
8310 static void
8311 action_build_free(struct rte_swx_pipeline *p)
8312 {
8313         free(p->action_instructions);
8314         p->action_instructions = NULL;
8315 }
8316
8317 static void
8318 action_free(struct rte_swx_pipeline *p)
8319 {
8320         action_build_free(p);
8321
8322         for ( ; ; ) {
8323                 struct action *action;
8324
8325                 action = TAILQ_FIRST(&p->actions);
8326                 if (!action)
8327                         break;
8328
8329                 TAILQ_REMOVE(&p->actions, action, node);
8330                 free(action->instructions);
8331                 free(action);
8332         }
8333 }
8334
8335 /*
8336  * Table.
8337  */
8338 static struct table_type *
8339 table_type_find(struct rte_swx_pipeline *p, const char *name)
8340 {
8341         struct table_type *elem;
8342
8343         TAILQ_FOREACH(elem, &p->table_types, node)
8344                 if (strcmp(elem->name, name) == 0)
8345                         return elem;
8346
8347         return NULL;
8348 }
8349
8350 static struct table_type *
8351 table_type_resolve(struct rte_swx_pipeline *p,
8352                    const char *recommended_type_name,
8353                    enum rte_swx_table_match_type match_type)
8354 {
8355         struct table_type *elem;
8356
8357         /* Only consider the recommended type if the match type is correct. */
8358         if (recommended_type_name)
8359                 TAILQ_FOREACH(elem, &p->table_types, node)
8360                         if (!strcmp(elem->name, recommended_type_name) &&
8361                             (elem->match_type == match_type))
8362                                 return elem;
8363
8364         /* Ignore the recommended type and get the first element with this match
8365          * type.
8366          */
8367         TAILQ_FOREACH(elem, &p->table_types, node)
8368                 if (elem->match_type == match_type)
8369                         return elem;
8370
8371         return NULL;
8372 }
8373
8374 static struct table *
8375 table_find(struct rte_swx_pipeline *p, const char *name)
8376 {
8377         struct table *elem;
8378
8379         TAILQ_FOREACH(elem, &p->tables, node)
8380                 if (strcmp(elem->name, name) == 0)
8381                         return elem;
8382
8383         return NULL;
8384 }
8385
8386 static struct table *
8387 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8388 {
8389         struct table *table = NULL;
8390
8391         TAILQ_FOREACH(table, &p->tables, node)
8392                 if (table->id == id)
8393                         return table;
8394
8395         return NULL;
8396 }
8397
8398 int
8399 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
8400                                      const char *name,
8401                                      enum rte_swx_table_match_type match_type,
8402                                      struct rte_swx_table_ops *ops)
8403 {
8404         struct table_type *elem;
8405
8406         CHECK(p, EINVAL);
8407
8408         CHECK_NAME(name, EINVAL);
8409         CHECK(!table_type_find(p, name), EEXIST);
8410
8411         CHECK(ops, EINVAL);
8412         CHECK(ops->create, EINVAL);
8413         CHECK(ops->lkp, EINVAL);
8414         CHECK(ops->free, EINVAL);
8415
8416         /* Node allocation. */
8417         elem = calloc(1, sizeof(struct table_type));
8418         CHECK(elem, ENOMEM);
8419
8420         /* Node initialization. */
8421         strcpy(elem->name, name);
8422         elem->match_type = match_type;
8423         memcpy(&elem->ops, ops, sizeof(*ops));
8424
8425         /* Node add to tailq. */
8426         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
8427
8428         return 0;
8429 }
8430
8431 static enum rte_swx_table_match_type
8432 table_match_type_resolve(struct rte_swx_match_field_params *fields,
8433                          uint32_t n_fields,
8434                          uint32_t max_offset_field_id)
8435 {
8436         uint32_t n_fields_em = 0, i;
8437
8438         for (i = 0; i < n_fields; i++)
8439                 if (fields[i].match_type == RTE_SWX_TABLE_MATCH_EXACT)
8440                         n_fields_em++;
8441
8442         if (n_fields_em == n_fields)
8443                 return RTE_SWX_TABLE_MATCH_EXACT;
8444
8445         if ((n_fields_em == n_fields - 1) &&
8446             (fields[max_offset_field_id].match_type == RTE_SWX_TABLE_MATCH_LPM))
8447                 return RTE_SWX_TABLE_MATCH_LPM;
8448
8449         return RTE_SWX_TABLE_MATCH_WILDCARD;
8450 }
8451
8452 static int
8453 table_match_fields_check(struct rte_swx_pipeline *p,
8454                          struct rte_swx_pipeline_table_params *params,
8455                          struct header **header,
8456                          uint32_t *min_offset_field_id,
8457                          uint32_t *max_offset_field_id)
8458 {
8459         struct header *h0 = NULL;
8460         struct field *hf, *mf;
8461         uint32_t *offset = NULL, min_offset, max_offset, min_offset_pos, max_offset_pos, i;
8462         int status = 0;
8463
8464         /* Return if no match fields. */
8465         if (!params->n_fields) {
8466                 if (params->fields) {
8467                         status = -EINVAL;
8468                         goto end;
8469                 }
8470
8471                 return 0;
8472         }
8473
8474         /* Memory allocation. */
8475         offset = calloc(params->n_fields, sizeof(uint32_t));
8476         if (!offset) {
8477                 status = -ENOMEM;
8478                 goto end;
8479         }
8480
8481         /* Check that all the match fields belong to either the same header or
8482          * to the meta-data.
8483          */
8484         hf = header_field_parse(p, params->fields[0].name, &h0);
8485         mf = metadata_field_parse(p, params->fields[0].name);
8486         if (!hf && !mf) {
8487                 status = -EINVAL;
8488                 goto end;
8489         }
8490
8491         offset[0] = h0 ? hf->offset : mf->offset;
8492
8493         for (i = 1; i < params->n_fields; i++)
8494                 if (h0) {
8495                         struct header *h;
8496
8497                         hf = header_field_parse(p, params->fields[i].name, &h);
8498                         if (!hf || (h->id != h0->id)) {
8499                                 status = -EINVAL;
8500                                 goto end;
8501                         }
8502
8503                         offset[i] = hf->offset;
8504                 } else {
8505                         mf = metadata_field_parse(p, params->fields[i].name);
8506                         if (!mf) {
8507                                 status = -EINVAL;
8508                                 goto end;
8509                         }
8510
8511                         offset[i] = mf->offset;
8512                 }
8513
8514         /* Check that there are no duplicated match fields. */
8515         for (i = 0; i < params->n_fields; i++) {
8516                 uint32_t j;
8517
8518                 for (j = 0; j < i; j++)
8519                         if (offset[j] == offset[i]) {
8520                                 status = -EINVAL;
8521                                 goto end;
8522                         }
8523         }
8524
8525         /* Find the min and max offset fields. */
8526         min_offset = offset[0];
8527         max_offset = offset[0];
8528         min_offset_pos = 0;
8529         max_offset_pos = 0;
8530
8531         for (i = 1; i < params->n_fields; i++) {
8532                 if (offset[i] < min_offset) {
8533                         min_offset = offset[i];
8534                         min_offset_pos = i;
8535                 }
8536
8537                 if (offset[i] > max_offset) {
8538                         max_offset = offset[i];
8539                         max_offset_pos = i;
8540                 }
8541         }
8542
8543         /* Return. */
8544         if (header)
8545                 *header = h0;
8546
8547         if (min_offset_field_id)
8548                 *min_offset_field_id = min_offset_pos;
8549
8550         if (max_offset_field_id)
8551                 *max_offset_field_id = max_offset_pos;
8552
8553 end:
8554         free(offset);
8555         return status;
8556 }
8557
8558 int
8559 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8560                               const char *name,
8561                               struct rte_swx_pipeline_table_params *params,
8562                               const char *recommended_table_type_name,
8563                               const char *args,
8564                               uint32_t size)
8565 {
8566         struct table_type *type;
8567         struct table *t;
8568         struct action *default_action;
8569         struct header *header = NULL;
8570         uint32_t action_data_size_max = 0, min_offset_field_id = 0, max_offset_field_id = 0, i;
8571         int status = 0;
8572
8573         CHECK(p, EINVAL);
8574
8575         CHECK_NAME(name, EINVAL);
8576         CHECK(!table_find(p, name), EEXIST);
8577
8578         CHECK(params, EINVAL);
8579
8580         /* Match checks. */
8581         status = table_match_fields_check(p,
8582                                           params,
8583                                           &header,
8584                                           &min_offset_field_id,
8585                                           &max_offset_field_id);
8586         if (status)
8587                 return status;
8588
8589         /* Action checks. */
8590         CHECK(params->n_actions, EINVAL);
8591         CHECK(params->action_names, EINVAL);
8592         for (i = 0; i < params->n_actions; i++) {
8593                 const char *action_name = params->action_names[i];
8594                 struct action *a;
8595                 uint32_t action_data_size;
8596
8597                 CHECK_NAME(action_name, EINVAL);
8598
8599                 a = action_find(p, action_name);
8600                 CHECK(a, EINVAL);
8601
8602                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8603                 if (action_data_size > action_data_size_max)
8604                         action_data_size_max = action_data_size;
8605         }
8606
8607         CHECK_NAME(params->default_action_name, EINVAL);
8608         for (i = 0; i < p->n_actions; i++)
8609                 if (!strcmp(params->action_names[i],
8610                             params->default_action_name))
8611                         break;
8612         CHECK(i < params->n_actions, EINVAL);
8613         default_action = action_find(p, params->default_action_name);
8614         CHECK((default_action->st && params->default_action_data) ||
8615               !params->default_action_data, EINVAL);
8616
8617         /* Table type checks. */
8618         if (recommended_table_type_name)
8619                 CHECK_NAME(recommended_table_type_name, EINVAL);
8620
8621         if (params->n_fields) {
8622                 enum rte_swx_table_match_type match_type;
8623
8624                 match_type = table_match_type_resolve(params->fields,
8625                                                       params->n_fields,
8626                                                       max_offset_field_id);
8627                 type = table_type_resolve(p,
8628                                           recommended_table_type_name,
8629                                           match_type);
8630                 CHECK(type, EINVAL);
8631         } else {
8632                 type = NULL;
8633         }
8634
8635         /* Memory allocation. */
8636         t = calloc(1, sizeof(struct table));
8637         CHECK(t, ENOMEM);
8638
8639         t->fields = calloc(params->n_fields, sizeof(struct match_field));
8640         if (!t->fields) {
8641                 free(t);
8642                 CHECK(0, ENOMEM);
8643         }
8644
8645         t->actions = calloc(params->n_actions, sizeof(struct action *));
8646         if (!t->actions) {
8647                 free(t->fields);
8648                 free(t);
8649                 CHECK(0, ENOMEM);
8650         }
8651
8652         if (action_data_size_max) {
8653                 t->default_action_data = calloc(1, action_data_size_max);
8654                 if (!t->default_action_data) {
8655                         free(t->actions);
8656                         free(t->fields);
8657                         free(t);
8658                         CHECK(0, ENOMEM);
8659                 }
8660         }
8661
8662         /* Node initialization. */
8663         strcpy(t->name, name);
8664         if (args && args[0])
8665                 strcpy(t->args, args);
8666         t->type = type;
8667
8668         for (i = 0; i < params->n_fields; i++) {
8669                 struct rte_swx_match_field_params *field = &params->fields[i];
8670                 struct match_field *f = &t->fields[i];
8671
8672                 f->match_type = field->match_type;
8673                 f->field = header ?
8674                         header_field_parse(p, field->name, NULL) :
8675                         metadata_field_parse(p, field->name);
8676         }
8677         t->n_fields = params->n_fields;
8678         t->header = header;
8679
8680         for (i = 0; i < params->n_actions; i++)
8681                 t->actions[i] = action_find(p, params->action_names[i]);
8682         t->default_action = default_action;
8683         if (default_action->st)
8684                 memcpy(t->default_action_data,
8685                        params->default_action_data,
8686                        default_action->st->n_bits / 8);
8687         t->n_actions = params->n_actions;
8688         t->default_action_is_const = params->default_action_is_const;
8689         t->action_data_size_max = action_data_size_max;
8690
8691         t->size = size;
8692         t->id = p->n_tables;
8693
8694         /* Node add to tailq. */
8695         TAILQ_INSERT_TAIL(&p->tables, t, node);
8696         p->n_tables++;
8697
8698         return 0;
8699 }
8700
8701 static struct rte_swx_table_params *
8702 table_params_get(struct table *table)
8703 {
8704         struct rte_swx_table_params *params;
8705         struct field *first, *last;
8706         uint8_t *key_mask;
8707         uint32_t key_size, key_offset, action_data_size, i;
8708
8709         /* Memory allocation. */
8710         params = calloc(1, sizeof(struct rte_swx_table_params));
8711         if (!params)
8712                 return NULL;
8713
8714         /* Find first (smallest offset) and last (biggest offset) match fields. */
8715         first = table->fields[0].field;
8716         last = table->fields[0].field;
8717
8718         for (i = 0; i < table->n_fields; i++) {
8719                 struct field *f = table->fields[i].field;
8720
8721                 if (f->offset < first->offset)
8722                         first = f;
8723
8724                 if (f->offset > last->offset)
8725                         last = f;
8726         }
8727
8728         /* Key offset and size. */
8729         key_offset = first->offset / 8;
8730         key_size = (last->offset + last->n_bits - first->offset) / 8;
8731
8732         /* Memory allocation. */
8733         key_mask = calloc(1, key_size);
8734         if (!key_mask) {
8735                 free(params);
8736                 return NULL;
8737         }
8738
8739         /* Key mask. */
8740         for (i = 0; i < table->n_fields; i++) {
8741                 struct field *f = table->fields[i].field;
8742                 uint32_t start = (f->offset - first->offset) / 8;
8743                 size_t size = f->n_bits / 8;
8744
8745                 memset(&key_mask[start], 0xFF, size);
8746         }
8747
8748         /* Action data size. */
8749         action_data_size = 0;
8750         for (i = 0; i < table->n_actions; i++) {
8751                 struct action *action = table->actions[i];
8752                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8753
8754                 if (ads > action_data_size)
8755                         action_data_size = ads;
8756         }
8757
8758         /* Fill in. */
8759         params->match_type = table->type->match_type;
8760         params->key_size = key_size;
8761         params->key_offset = key_offset;
8762         params->key_mask0 = key_mask;
8763         params->action_data_size = action_data_size;
8764         params->n_keys_max = table->size;
8765
8766         return params;
8767 }
8768
8769 static void
8770 table_params_free(struct rte_swx_table_params *params)
8771 {
8772         if (!params)
8773                 return;
8774
8775         free(params->key_mask0);
8776         free(params);
8777 }
8778
8779 static int
8780 table_state_build(struct rte_swx_pipeline *p)
8781 {
8782         struct table *table;
8783
8784         p->table_state = calloc(p->n_tables,
8785                                 sizeof(struct rte_swx_table_state));
8786         CHECK(p->table_state, ENOMEM);
8787
8788         TAILQ_FOREACH(table, &p->tables, node) {
8789                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8790
8791                 if (table->type) {
8792                         struct rte_swx_table_params *params;
8793
8794                         /* ts->obj. */
8795                         params = table_params_get(table);
8796                         CHECK(params, ENOMEM);
8797
8798                         ts->obj = table->type->ops.create(params,
8799                                 NULL,
8800                                 table->args,
8801                                 p->numa_node);
8802
8803                         table_params_free(params);
8804                         CHECK(ts->obj, ENODEV);
8805                 }
8806
8807                 /* ts->default_action_data. */
8808                 if (table->action_data_size_max) {
8809                         ts->default_action_data =
8810                                 malloc(table->action_data_size_max);
8811                         CHECK(ts->default_action_data, ENOMEM);
8812
8813                         memcpy(ts->default_action_data,
8814                                table->default_action_data,
8815                                table->action_data_size_max);
8816                 }
8817
8818                 /* ts->default_action_id. */
8819                 ts->default_action_id = table->default_action->id;
8820         }
8821
8822         return 0;
8823 }
8824
8825 static void
8826 table_state_build_free(struct rte_swx_pipeline *p)
8827 {
8828         uint32_t i;
8829
8830         if (!p->table_state)
8831                 return;
8832
8833         for (i = 0; i < p->n_tables; i++) {
8834                 struct rte_swx_table_state *ts = &p->table_state[i];
8835                 struct table *table = table_find_by_id(p, i);
8836
8837                 /* ts->obj. */
8838                 if (table->type && ts->obj)
8839                         table->type->ops.free(ts->obj);
8840
8841                 /* ts->default_action_data. */
8842                 free(ts->default_action_data);
8843         }
8844
8845         free(p->table_state);
8846         p->table_state = NULL;
8847 }
8848
8849 static void
8850 table_state_free(struct rte_swx_pipeline *p)
8851 {
8852         table_state_build_free(p);
8853 }
8854
8855 static int
8856 table_stub_lkp(void *table __rte_unused,
8857                void *mailbox __rte_unused,
8858                uint8_t **key __rte_unused,
8859                uint64_t *action_id __rte_unused,
8860                uint8_t **action_data __rte_unused,
8861                int *hit)
8862 {
8863         *hit = 0;
8864         return 1; /* DONE. */
8865 }
8866
8867 static int
8868 table_build(struct rte_swx_pipeline *p)
8869 {
8870         uint32_t i;
8871
8872         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8873                 struct thread *t = &p->threads[i];
8874                 struct table *table;
8875
8876                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8877                 CHECK(t->tables, ENOMEM);
8878
8879                 TAILQ_FOREACH(table, &p->tables, node) {
8880                         struct table_runtime *r = &t->tables[table->id];
8881
8882                         if (table->type) {
8883                                 uint64_t size;
8884
8885                                 size = table->type->ops.mailbox_size_get();
8886
8887                                 /* r->func. */
8888                                 r->func = table->type->ops.lkp;
8889
8890                                 /* r->mailbox. */
8891                                 if (size) {
8892                                         r->mailbox = calloc(1, size);
8893                                         CHECK(r->mailbox, ENOMEM);
8894                                 }
8895
8896                                 /* r->key. */
8897                                 r->key = table->header ?
8898                                         &t->structs[table->header->struct_id] :
8899                                         &t->structs[p->metadata_struct_id];
8900                         } else {
8901                                 r->func = table_stub_lkp;
8902                         }
8903                 }
8904         }
8905
8906         return 0;
8907 }
8908
8909 static void
8910 table_build_free(struct rte_swx_pipeline *p)
8911 {
8912         uint32_t i;
8913
8914         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8915                 struct thread *t = &p->threads[i];
8916                 uint32_t j;
8917
8918                 if (!t->tables)
8919                         continue;
8920
8921                 for (j = 0; j < p->n_tables; j++) {
8922                         struct table_runtime *r = &t->tables[j];
8923
8924                         free(r->mailbox);
8925                 }
8926
8927                 free(t->tables);
8928                 t->tables = NULL;
8929         }
8930 }
8931
8932 static void
8933 table_free(struct rte_swx_pipeline *p)
8934 {
8935         table_build_free(p);
8936
8937         /* Tables. */
8938         for ( ; ; ) {
8939                 struct table *elem;
8940
8941                 elem = TAILQ_FIRST(&p->tables);
8942                 if (!elem)
8943                         break;
8944
8945                 TAILQ_REMOVE(&p->tables, elem, node);
8946                 free(elem->fields);
8947                 free(elem->actions);
8948                 free(elem->default_action_data);
8949                 free(elem);
8950         }
8951
8952         /* Table types. */
8953         for ( ; ; ) {
8954                 struct table_type *elem;
8955
8956                 elem = TAILQ_FIRST(&p->table_types);
8957                 if (!elem)
8958                         break;
8959
8960                 TAILQ_REMOVE(&p->table_types, elem, node);
8961                 free(elem);
8962         }
8963 }
8964
8965 /*
8966  * Register array.
8967  */
8968 static struct regarray *
8969 regarray_find(struct rte_swx_pipeline *p, const char *name)
8970 {
8971         struct regarray *elem;
8972
8973         TAILQ_FOREACH(elem, &p->regarrays, node)
8974                 if (!strcmp(elem->name, name))
8975                         return elem;
8976
8977         return NULL;
8978 }
8979
8980 static struct regarray *
8981 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8982 {
8983         struct regarray *elem = NULL;
8984
8985         TAILQ_FOREACH(elem, &p->regarrays, node)
8986                 if (elem->id == id)
8987                         return elem;
8988
8989         return NULL;
8990 }
8991
8992 int
8993 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8994                               const char *name,
8995                               uint32_t size,
8996                               uint64_t init_val)
8997 {
8998         struct regarray *r;
8999
9000         CHECK(p, EINVAL);
9001
9002         CHECK_NAME(name, EINVAL);
9003         CHECK(!regarray_find(p, name), EEXIST);
9004
9005         CHECK(size, EINVAL);
9006         size = rte_align32pow2(size);
9007
9008         /* Memory allocation. */
9009         r = calloc(1, sizeof(struct regarray));
9010         CHECK(r, ENOMEM);
9011
9012         /* Node initialization. */
9013         strcpy(r->name, name);
9014         r->init_val = init_val;
9015         r->size = size;
9016         r->id = p->n_regarrays;
9017
9018         /* Node add to tailq. */
9019         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9020         p->n_regarrays++;
9021
9022         return 0;
9023 }
9024
9025 static int
9026 regarray_build(struct rte_swx_pipeline *p)
9027 {
9028         struct regarray *regarray;
9029
9030         if (!p->n_regarrays)
9031                 return 0;
9032
9033         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9034         CHECK(p->regarray_runtime, ENOMEM);
9035
9036         TAILQ_FOREACH(regarray, &p->regarrays, node) {
9037                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9038                 uint32_t i;
9039
9040                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9041                                          RTE_CACHE_LINE_SIZE,
9042                                          p->numa_node);
9043                 CHECK(r->regarray, ENOMEM);
9044
9045                 if (regarray->init_val)
9046                         for (i = 0; i < regarray->size; i++)
9047                                 r->regarray[i] = regarray->init_val;
9048
9049                 r->size_mask = regarray->size - 1;
9050         }
9051
9052         return 0;
9053 }
9054
9055 static void
9056 regarray_build_free(struct rte_swx_pipeline *p)
9057 {
9058         uint32_t i;
9059
9060         if (!p->regarray_runtime)
9061                 return;
9062
9063         for (i = 0; i < p->n_regarrays; i++) {
9064                 struct regarray *regarray = regarray_find_by_id(p, i);
9065                 struct regarray_runtime *r = &p->regarray_runtime[i];
9066
9067                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9068         }
9069
9070         free(p->regarray_runtime);
9071         p->regarray_runtime = NULL;
9072 }
9073
9074 static void
9075 regarray_free(struct rte_swx_pipeline *p)
9076 {
9077         regarray_build_free(p);
9078
9079         for ( ; ; ) {
9080                 struct regarray *elem;
9081
9082                 elem = TAILQ_FIRST(&p->regarrays);
9083                 if (!elem)
9084                         break;
9085
9086                 TAILQ_REMOVE(&p->regarrays, elem, node);
9087                 free(elem);
9088         }
9089 }
9090
9091 /*
9092  * Meter array.
9093  */
9094 static struct meter_profile *
9095 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9096 {
9097         struct meter_profile *elem;
9098
9099         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9100                 if (!strcmp(elem->name, name))
9101                         return elem;
9102
9103         return NULL;
9104 }
9105
9106 static struct metarray *
9107 metarray_find(struct rte_swx_pipeline *p, const char *name)
9108 {
9109         struct metarray *elem;
9110
9111         TAILQ_FOREACH(elem, &p->metarrays, node)
9112                 if (!strcmp(elem->name, name))
9113                         return elem;
9114
9115         return NULL;
9116 }
9117
9118 static struct metarray *
9119 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9120 {
9121         struct metarray *elem = NULL;
9122
9123         TAILQ_FOREACH(elem, &p->metarrays, node)
9124                 if (elem->id == id)
9125                         return elem;
9126
9127         return NULL;
9128 }
9129
9130 int
9131 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9132                                  const char *name,
9133                                  uint32_t size)
9134 {
9135         struct metarray *m;
9136
9137         CHECK(p, EINVAL);
9138
9139         CHECK_NAME(name, EINVAL);
9140         CHECK(!metarray_find(p, name), EEXIST);
9141
9142         CHECK(size, EINVAL);
9143         size = rte_align32pow2(size);
9144
9145         /* Memory allocation. */
9146         m = calloc(1, sizeof(struct metarray));
9147         CHECK(m, ENOMEM);
9148
9149         /* Node initialization. */
9150         strcpy(m->name, name);
9151         m->size = size;
9152         m->id = p->n_metarrays;
9153
9154         /* Node add to tailq. */
9155         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9156         p->n_metarrays++;
9157
9158         return 0;
9159 }
9160
9161 struct meter_profile meter_profile_default = {
9162         .node = {0},
9163         .name = "",
9164         .params = {0},
9165
9166         .profile = {
9167                 .cbs = 10000,
9168                 .pbs = 10000,
9169                 .cir_period = 1,
9170                 .cir_bytes_per_period = 1,
9171                 .pir_period = 1,
9172                 .pir_bytes_per_period = 1,
9173         },
9174
9175         .n_users = 0,
9176 };
9177
9178 static void
9179 meter_init(struct meter *m)
9180 {
9181         memset(m, 0, sizeof(struct meter));
9182         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9183         m->profile = &meter_profile_default;
9184         m->color_mask = RTE_COLOR_GREEN;
9185
9186         meter_profile_default.n_users++;
9187 }
9188
9189 static int
9190 metarray_build(struct rte_swx_pipeline *p)
9191 {
9192         struct metarray *m;
9193
9194         if (!p->n_metarrays)
9195                 return 0;
9196
9197         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9198         CHECK(p->metarray_runtime, ENOMEM);
9199
9200         TAILQ_FOREACH(m, &p->metarrays, node) {
9201                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9202                 uint32_t i;
9203
9204                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9205                                          RTE_CACHE_LINE_SIZE,
9206                                          p->numa_node);
9207                 CHECK(r->metarray, ENOMEM);
9208
9209                 for (i = 0; i < m->size; i++)
9210                         meter_init(&r->metarray[i]);
9211
9212                 r->size_mask = m->size - 1;
9213         }
9214
9215         return 0;
9216 }
9217
9218 static void
9219 metarray_build_free(struct rte_swx_pipeline *p)
9220 {
9221         uint32_t i;
9222
9223         if (!p->metarray_runtime)
9224                 return;
9225
9226         for (i = 0; i < p->n_metarrays; i++) {
9227                 struct metarray *m = metarray_find_by_id(p, i);
9228                 struct metarray_runtime *r = &p->metarray_runtime[i];
9229
9230                 env_free(r->metarray, m->size * sizeof(struct meter));
9231         }
9232
9233         free(p->metarray_runtime);
9234         p->metarray_runtime = NULL;
9235 }
9236
9237 static void
9238 metarray_free(struct rte_swx_pipeline *p)
9239 {
9240         metarray_build_free(p);
9241
9242         /* Meter arrays. */
9243         for ( ; ; ) {
9244                 struct metarray *elem;
9245
9246                 elem = TAILQ_FIRST(&p->metarrays);
9247                 if (!elem)
9248                         break;
9249
9250                 TAILQ_REMOVE(&p->metarrays, elem, node);
9251                 free(elem);
9252         }
9253
9254         /* Meter profiles. */
9255         for ( ; ; ) {
9256                 struct meter_profile *elem;
9257
9258                 elem = TAILQ_FIRST(&p->meter_profiles);
9259                 if (!elem)
9260                         break;
9261
9262                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9263                 free(elem);
9264         }
9265 }
9266
9267 /*
9268  * Pipeline.
9269  */
9270 int
9271 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9272 {
9273         struct rte_swx_pipeline *pipeline;
9274
9275         /* Check input parameters. */
9276         CHECK(p, EINVAL);
9277
9278         /* Memory allocation. */
9279         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9280         CHECK(pipeline, ENOMEM);
9281
9282         /* Initialization. */
9283         TAILQ_INIT(&pipeline->struct_types);
9284         TAILQ_INIT(&pipeline->port_in_types);
9285         TAILQ_INIT(&pipeline->ports_in);
9286         TAILQ_INIT(&pipeline->port_out_types);
9287         TAILQ_INIT(&pipeline->ports_out);
9288         TAILQ_INIT(&pipeline->extern_types);
9289         TAILQ_INIT(&pipeline->extern_objs);
9290         TAILQ_INIT(&pipeline->extern_funcs);
9291         TAILQ_INIT(&pipeline->headers);
9292         TAILQ_INIT(&pipeline->actions);
9293         TAILQ_INIT(&pipeline->table_types);
9294         TAILQ_INIT(&pipeline->tables);
9295         TAILQ_INIT(&pipeline->regarrays);
9296         TAILQ_INIT(&pipeline->meter_profiles);
9297         TAILQ_INIT(&pipeline->metarrays);
9298
9299         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9300         pipeline->numa_node = numa_node;
9301
9302         *p = pipeline;
9303         return 0;
9304 }
9305
9306 void
9307 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9308 {
9309         if (!p)
9310                 return;
9311
9312         free(p->instructions);
9313
9314         metarray_free(p);
9315         regarray_free(p);
9316         table_state_free(p);
9317         table_free(p);
9318         action_free(p);
9319         metadata_free(p);
9320         header_free(p);
9321         extern_func_free(p);
9322         extern_obj_free(p);
9323         port_out_free(p);
9324         port_in_free(p);
9325         struct_free(p);
9326
9327         free(p);
9328 }
9329
9330 int
9331 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9332                                      const char **instructions,
9333                                      uint32_t n_instructions)
9334 {
9335         int err;
9336         uint32_t i;
9337
9338         err = instruction_config(p, NULL, instructions, n_instructions);
9339         if (err)
9340                 return err;
9341
9342         /* Thread instruction pointer reset. */
9343         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9344                 struct thread *t = &p->threads[i];
9345
9346                 thread_ip_reset(p, t);
9347         }
9348
9349         return 0;
9350 }
9351
9352 int
9353 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9354 {
9355         int status;
9356
9357         CHECK(p, EINVAL);
9358         CHECK(p->build_done == 0, EEXIST);
9359
9360         status = port_in_build(p);
9361         if (status)
9362                 goto error;
9363
9364         status = port_out_build(p);
9365         if (status)
9366                 goto error;
9367
9368         status = struct_build(p);
9369         if (status)
9370                 goto error;
9371
9372         status = extern_obj_build(p);
9373         if (status)
9374                 goto error;
9375
9376         status = extern_func_build(p);
9377         if (status)
9378                 goto error;
9379
9380         status = header_build(p);
9381         if (status)
9382                 goto error;
9383
9384         status = metadata_build(p);
9385         if (status)
9386                 goto error;
9387
9388         status = action_build(p);
9389         if (status)
9390                 goto error;
9391
9392         status = table_build(p);
9393         if (status)
9394                 goto error;
9395
9396         status = table_state_build(p);
9397         if (status)
9398                 goto error;
9399
9400         status = regarray_build(p);
9401         if (status)
9402                 goto error;
9403
9404         status = metarray_build(p);
9405         if (status)
9406                 goto error;
9407
9408         p->build_done = 1;
9409         return 0;
9410
9411 error:
9412         metarray_build_free(p);
9413         regarray_build_free(p);
9414         table_state_build_free(p);
9415         table_build_free(p);
9416         action_build_free(p);
9417         metadata_build_free(p);
9418         header_build_free(p);
9419         extern_func_build_free(p);
9420         extern_obj_build_free(p);
9421         port_out_build_free(p);
9422         port_in_build_free(p);
9423         struct_build_free(p);
9424
9425         return status;
9426 }
9427
9428 void
9429 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9430 {
9431         uint32_t i;
9432
9433         for (i = 0; i < n_instructions; i++)
9434                 instr_exec(p);
9435 }
9436
9437 void
9438 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9439 {
9440         uint32_t i;
9441
9442         for (i = 0; i < p->n_ports_out; i++) {
9443                 struct port_out_runtime *port = &p->out[i];
9444
9445                 if (port->flush)
9446                         port->flush(port->obj);
9447         }
9448 }
9449
9450 /*
9451  * Control.
9452  */
9453 int
9454 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9455                               struct rte_swx_ctl_pipeline_info *pipeline)
9456 {
9457         struct action *action;
9458         struct table *table;
9459         uint32_t n_actions = 0, n_tables = 0;
9460
9461         if (!p || !pipeline)
9462                 return -EINVAL;
9463
9464         TAILQ_FOREACH(action, &p->actions, node)
9465                 n_actions++;
9466
9467         TAILQ_FOREACH(table, &p->tables, node)
9468                 n_tables++;
9469
9470         pipeline->n_ports_in = p->n_ports_in;
9471         pipeline->n_ports_out = p->n_ports_out;
9472         pipeline->n_actions = n_actions;
9473         pipeline->n_tables = n_tables;
9474         pipeline->n_regarrays = p->n_regarrays;
9475         pipeline->n_metarrays = p->n_metarrays;
9476
9477         return 0;
9478 }
9479
9480 int
9481 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9482 {
9483         if (!p || !numa_node)
9484                 return -EINVAL;
9485
9486         *numa_node = p->numa_node;
9487         return 0;
9488 }
9489
9490 int
9491 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9492                             uint32_t action_id,
9493                             struct rte_swx_ctl_action_info *action)
9494 {
9495         struct action *a = NULL;
9496
9497         if (!p || (action_id >= p->n_actions) || !action)
9498                 return -EINVAL;
9499
9500         a = action_find_by_id(p, action_id);
9501         if (!a)
9502                 return -EINVAL;
9503
9504         strcpy(action->name, a->name);
9505         action->n_args = a->st ? a->st->n_fields : 0;
9506         return 0;
9507 }
9508
9509 int
9510 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9511                                 uint32_t action_id,
9512                                 uint32_t action_arg_id,
9513                                 struct rte_swx_ctl_action_arg_info *action_arg)
9514 {
9515         struct action *a = NULL;
9516         struct field *arg = NULL;
9517
9518         if (!p || (action_id >= p->n_actions) || !action_arg)
9519                 return -EINVAL;
9520
9521         a = action_find_by_id(p, action_id);
9522         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9523                 return -EINVAL;
9524
9525         arg = &a->st->fields[action_arg_id];
9526         strcpy(action_arg->name, arg->name);
9527         action_arg->n_bits = arg->n_bits;
9528
9529         return 0;
9530 }
9531
9532 int
9533 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9534                            uint32_t table_id,
9535                            struct rte_swx_ctl_table_info *table)
9536 {
9537         struct table *t = NULL;
9538
9539         if (!p || !table)
9540                 return -EINVAL;
9541
9542         t = table_find_by_id(p, table_id);
9543         if (!t)
9544                 return -EINVAL;
9545
9546         strcpy(table->name, t->name);
9547         strcpy(table->args, t->args);
9548         table->n_match_fields = t->n_fields;
9549         table->n_actions = t->n_actions;
9550         table->default_action_is_const = t->default_action_is_const;
9551         table->size = t->size;
9552         return 0;
9553 }
9554
9555 int
9556 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9557         uint32_t table_id,
9558         uint32_t match_field_id,
9559         struct rte_swx_ctl_table_match_field_info *match_field)
9560 {
9561         struct table *t;
9562         struct match_field *f;
9563
9564         if (!p || (table_id >= p->n_tables) || !match_field)
9565                 return -EINVAL;
9566
9567         t = table_find_by_id(p, table_id);
9568         if (!t || (match_field_id >= t->n_fields))
9569                 return -EINVAL;
9570
9571         f = &t->fields[match_field_id];
9572         match_field->match_type = f->match_type;
9573         match_field->is_header = t->header ? 1 : 0;
9574         match_field->n_bits = f->field->n_bits;
9575         match_field->offset = f->field->offset;
9576
9577         return 0;
9578 }
9579
9580 int
9581 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9582         uint32_t table_id,
9583         uint32_t table_action_id,
9584         struct rte_swx_ctl_table_action_info *table_action)
9585 {
9586         struct table *t;
9587
9588         if (!p || (table_id >= p->n_tables) || !table_action)
9589                 return -EINVAL;
9590
9591         t = table_find_by_id(p, table_id);
9592         if (!t || (table_action_id >= t->n_actions))
9593                 return -EINVAL;
9594
9595         table_action->action_id = t->actions[table_action_id]->id;
9596
9597         return 0;
9598 }
9599
9600 int
9601 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9602                           uint32_t table_id,
9603                           struct rte_swx_table_ops *table_ops,
9604                           int *is_stub)
9605 {
9606         struct table *t;
9607
9608         if (!p || (table_id >= p->n_tables))
9609                 return -EINVAL;
9610
9611         t = table_find_by_id(p, table_id);
9612         if (!t)
9613                 return -EINVAL;
9614
9615         if (t->type) {
9616                 if (table_ops)
9617                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9618                 *is_stub = 0;
9619         } else {
9620                 *is_stub = 1;
9621         }
9622
9623         return 0;
9624 }
9625
9626 int
9627 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9628                                  struct rte_swx_table_state **table_state)
9629 {
9630         if (!p || !table_state || !p->build_done)
9631                 return -EINVAL;
9632
9633         *table_state = p->table_state;
9634         return 0;
9635 }
9636
9637 int
9638 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9639                                  struct rte_swx_table_state *table_state)
9640 {
9641         if (!p || !table_state || !p->build_done)
9642                 return -EINVAL;
9643
9644         p->table_state = table_state;
9645         return 0;
9646 }
9647
9648 int
9649 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9650                                         uint32_t port_id,
9651                                         struct rte_swx_port_in_stats *stats)
9652 {
9653         struct port_in *port;
9654
9655         if (!p || !stats)
9656                 return -EINVAL;
9657
9658         port = port_in_find(p, port_id);
9659         if (!port)
9660                 return -EINVAL;
9661
9662         port->type->ops.stats_read(port->obj, stats);
9663         return 0;
9664 }
9665
9666 int
9667 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9668                                          uint32_t port_id,
9669                                          struct rte_swx_port_out_stats *stats)
9670 {
9671         struct port_out *port;
9672
9673         if (!p || !stats)
9674                 return -EINVAL;
9675
9676         port = port_out_find(p, port_id);
9677         if (!port)
9678                 return -EINVAL;
9679
9680         port->type->ops.stats_read(port->obj, stats);
9681         return 0;
9682 }
9683
9684 int
9685 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9686                               uint32_t regarray_id,
9687                               struct rte_swx_ctl_regarray_info *regarray)
9688 {
9689         struct regarray *r;
9690
9691         if (!p || !regarray)
9692                 return -EINVAL;
9693
9694         r = regarray_find_by_id(p, regarray_id);
9695         if (!r)
9696                 return -EINVAL;
9697
9698         strcpy(regarray->name, r->name);
9699         regarray->size = r->size;
9700         return 0;
9701 }
9702
9703 int
9704 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9705                                    const char *regarray_name,
9706                                    uint32_t regarray_index,
9707                                    uint64_t *value)
9708 {
9709         struct regarray *regarray;
9710         struct regarray_runtime *r;
9711
9712         if (!p || !regarray_name || !value)
9713                 return -EINVAL;
9714
9715         regarray = regarray_find(p, regarray_name);
9716         if (!regarray || (regarray_index >= regarray->size))
9717                 return -EINVAL;
9718
9719         r = &p->regarray_runtime[regarray->id];
9720         *value = r->regarray[regarray_index];
9721         return 0;
9722 }
9723
9724 int
9725 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9726                                    const char *regarray_name,
9727                                    uint32_t regarray_index,
9728                                    uint64_t value)
9729 {
9730         struct regarray *regarray;
9731         struct regarray_runtime *r;
9732
9733         if (!p || !regarray_name)
9734                 return -EINVAL;
9735
9736         regarray = regarray_find(p, regarray_name);
9737         if (!regarray || (regarray_index >= regarray->size))
9738                 return -EINVAL;
9739
9740         r = &p->regarray_runtime[regarray->id];
9741         r->regarray[regarray_index] = value;
9742         return 0;
9743 }
9744
9745 int
9746 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9747                               uint32_t metarray_id,
9748                               struct rte_swx_ctl_metarray_info *metarray)
9749 {
9750         struct metarray *m;
9751
9752         if (!p || !metarray)
9753                 return -EINVAL;
9754
9755         m = metarray_find_by_id(p, metarray_id);
9756         if (!m)
9757                 return -EINVAL;
9758
9759         strcpy(metarray->name, m->name);
9760         metarray->size = m->size;
9761         return 0;
9762 }
9763
9764 int
9765 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9766                               const char *name,
9767                               struct rte_meter_trtcm_params *params)
9768 {
9769         struct meter_profile *mp;
9770         int status;
9771
9772         CHECK(p, EINVAL);
9773         CHECK_NAME(name, EINVAL);
9774         CHECK(params, EINVAL);
9775         CHECK(!meter_profile_find(p, name), EEXIST);
9776
9777         /* Node allocation. */
9778         mp = calloc(1, sizeof(struct meter_profile));
9779         CHECK(mp, ENOMEM);
9780
9781         /* Node initialization. */
9782         strcpy(mp->name, name);
9783         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9784         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9785         if (status) {
9786                 free(mp);
9787                 CHECK(0, EINVAL);
9788         }
9789
9790         /* Node add to tailq. */
9791         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9792
9793         return 0;
9794 }
9795
9796 int
9797 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9798                                  const char *name)
9799 {
9800         struct meter_profile *mp;
9801
9802         CHECK(p, EINVAL);
9803         CHECK_NAME(name, EINVAL);
9804
9805         mp = meter_profile_find(p, name);
9806         CHECK(mp, EINVAL);
9807         CHECK(!mp->n_users, EBUSY);
9808
9809         /* Remove node from tailq. */
9810         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9811         free(mp);
9812
9813         return 0;
9814 }
9815
9816 int
9817 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9818                         const char *metarray_name,
9819                         uint32_t metarray_index)
9820 {
9821         struct meter_profile *mp_old;
9822         struct metarray *metarray;
9823         struct metarray_runtime *metarray_runtime;
9824         struct meter *m;
9825
9826         CHECK(p, EINVAL);
9827         CHECK_NAME(metarray_name, EINVAL);
9828
9829         metarray = metarray_find(p, metarray_name);
9830         CHECK(metarray, EINVAL);
9831         CHECK(metarray_index < metarray->size, EINVAL);
9832
9833         metarray_runtime = &p->metarray_runtime[metarray->id];
9834         m = &metarray_runtime->metarray[metarray_index];
9835         mp_old = m->profile;
9836
9837         meter_init(m);
9838
9839         mp_old->n_users--;
9840
9841         return 0;
9842 }
9843
9844 int
9845 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9846                       const char *metarray_name,
9847                       uint32_t metarray_index,
9848                       const char *profile_name)
9849 {
9850         struct meter_profile *mp, *mp_old;
9851         struct metarray *metarray;
9852         struct metarray_runtime *metarray_runtime;
9853         struct meter *m;
9854
9855         CHECK(p, EINVAL);
9856         CHECK_NAME(metarray_name, EINVAL);
9857
9858         metarray = metarray_find(p, metarray_name);
9859         CHECK(metarray, EINVAL);
9860         CHECK(metarray_index < metarray->size, EINVAL);
9861
9862         mp = meter_profile_find(p, profile_name);
9863         CHECK(mp, EINVAL);
9864
9865         metarray_runtime = &p->metarray_runtime[metarray->id];
9866         m = &metarray_runtime->metarray[metarray_index];
9867         mp_old = m->profile;
9868
9869         memset(m, 0, sizeof(struct meter));
9870         rte_meter_trtcm_config(&m->m, &mp->profile);
9871         m->profile = mp;
9872         m->color_mask = RTE_COLORS;
9873
9874         mp->n_users++;
9875         mp_old->n_users--;
9876
9877         return 0;
9878 }
9879
9880 int
9881 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9882                              const char *metarray_name,
9883                              uint32_t metarray_index,
9884                              struct rte_swx_ctl_meter_stats *stats)
9885 {
9886         struct metarray *metarray;
9887         struct metarray_runtime *metarray_runtime;
9888         struct meter *m;
9889
9890         CHECK(p, EINVAL);
9891         CHECK_NAME(metarray_name, EINVAL);
9892
9893         metarray = metarray_find(p, metarray_name);
9894         CHECK(metarray, EINVAL);
9895         CHECK(metarray_index < metarray->size, EINVAL);
9896
9897         CHECK(stats, EINVAL);
9898
9899         metarray_runtime = &p->metarray_runtime[metarray->id];
9900         m = &metarray_runtime->metarray[metarray_index];
9901
9902         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9903         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9904
9905         return 0;
9906 }