lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <inttypes.h>
   9 #include <sys/queue.h>
  10 #include <arpa/inet.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_prefetch.h>
  14 #include <rte_byteorder.h>
  15 #include <rte_cycles.h>
  16 #include <rte_meter.h>
  17
  18 #include "rte_swx_pipeline.h"
  19 #include "rte_swx_ctl.h"
  20
  21 #define CHECK(condition, err_code)                                             \
  22 do {                                                                           \
  23         if (!(condition))                                                      \
  24                 return -(err_code);                                            \
  25 } while (0)
  26
  27 #define CHECK_NAME(name, err_code)                                             \
  28         CHECK((name) &&                                                        \
  29               (name)[0] &&                                                     \
  30               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  31               err_code)
  32
  33 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  34         CHECK((instr) &&                                                       \
  35               (instr)[0] &&                                                    \
  36               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  37                RTE_SWX_INSTRUCTION_SIZE),                                      \
  38               err_code)
  39
  40 #ifndef TRACE_LEVEL
  41 #define TRACE_LEVEL 0
  42 #endif
  43
  44 #if TRACE_LEVEL
  45 #define TRACE(...) printf(__VA_ARGS__)
  46 #else
  47 #define TRACE(...)
  48 #endif
  49
  50 /*
  51  * Environment.
  52  */
  53 #define ntoh64(x) rte_be_to_cpu_64(x)
  54 #define hton64(x) rte_cpu_to_be_64(x)
  55
  56 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  57
  58 #include <rte_malloc.h>
  59
  60 static void *
  61 env_malloc(size_t size, size_t alignment, int numa_node)
  62 {
  63         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  64 }
  65
  66 static void
  67 env_free(void *start, size_t size __rte_unused)
  68 {
  69         rte_free(start);
  70 }
  71
  72 #else
  73
  74 #include <numa.h>
  75
  76 static void *
  77 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  78 {
  79         void *start;
  80
  81         if (numa_available() == -1)
  82                 return NULL;
  83
  84         start = numa_alloc_onnode(size, numa_node);
  85         if (!start)
  86                 return NULL;
  87
  88         memset(start, 0, size);
  89         return start;
  90 }
  91
  92 static void
  93 env_free(void *start, size_t size)
  94 {
  95         if (numa_available() == -1)
  96                 return;
  97
  98         numa_free(start, size);
  99 }
 100
 101 #endif
 102
 103 /*
 104  * Struct.
 105  */
 106 struct field {
 107         char name[RTE_SWX_NAME_SIZE];
 108         uint32_t n_bits;
 109         uint32_t offset;
 110 };
 111
 112 struct struct_type {
 113         TAILQ_ENTRY(struct_type) node;
 114         char name[RTE_SWX_NAME_SIZE];
 115         struct field *fields;
 116         uint32_t n_fields;
 117         uint32_t n_bits;
 118 };
 119
 120 TAILQ_HEAD(struct_type_tailq, struct_type);
 121
 122 /*
 123  * Input port.
 124  */
 125 struct port_in_type {
 126         TAILQ_ENTRY(port_in_type) node;
 127         char name[RTE_SWX_NAME_SIZE];
 128         struct rte_swx_port_in_ops ops;
 129 };
 130
 131 TAILQ_HEAD(port_in_type_tailq, port_in_type);
 132
 133 struct port_in {
 134         TAILQ_ENTRY(port_in) node;
 135         struct port_in_type *type;
 136         void *obj;
 137         uint32_t id;
 138 };
 139
 140 TAILQ_HEAD(port_in_tailq, port_in);
 141
 142 struct port_in_runtime {
 143         rte_swx_port_in_pkt_rx_t pkt_rx;
 144         void *obj;
 145 };
 146
 147 /*
 148  * Output port.
 149  */
 150 struct port_out_type {
 151         TAILQ_ENTRY(port_out_type) node;
 152         char name[RTE_SWX_NAME_SIZE];
 153         struct rte_swx_port_out_ops ops;
 154 };
 155
 156 TAILQ_HEAD(port_out_type_tailq, port_out_type);
 157
 158 struct port_out {
 159         TAILQ_ENTRY(port_out) node;
 160         struct port_out_type *type;
 161         void *obj;
 162         uint32_t id;
 163 };
 164
 165 TAILQ_HEAD(port_out_tailq, port_out);
 166
 167 struct port_out_runtime {
 168         rte_swx_port_out_pkt_tx_t pkt_tx;
 169         rte_swx_port_out_flush_t flush;
 170         void *obj;
 171 };
 172
 173 /*
 174  * Extern object.
 175  */
 176 struct extern_type_member_func {
 177         TAILQ_ENTRY(extern_type_member_func) node;
 178         char name[RTE_SWX_NAME_SIZE];
 179         rte_swx_extern_type_member_func_t func;
 180         uint32_t id;
 181 };
 182
 183 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 184
 185 struct extern_type {
 186         TAILQ_ENTRY(extern_type) node;
 187         char name[RTE_SWX_NAME_SIZE];
 188         struct struct_type *mailbox_struct_type;
 189         rte_swx_extern_type_constructor_t constructor;
 190         rte_swx_extern_type_destructor_t destructor;
 191         struct extern_type_member_func_tailq funcs;
 192         uint32_t n_funcs;
 193 };
 194
 195 TAILQ_HEAD(extern_type_tailq, extern_type);
 196
 197 struct extern_obj {
 198         TAILQ_ENTRY(extern_obj) node;
 199         char name[RTE_SWX_NAME_SIZE];
 200         struct extern_type *type;
 201         void *obj;
 202         uint32_t struct_id;
 203         uint32_t id;
 204 };
 205
 206 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 207
 208 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 209 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 210 #endif
 211
 212 struct extern_obj_runtime {
 213         void *obj;
 214         uint8_t *mailbox;
 215         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 216 };
 217
 218 /*
 219  * Extern function.
 220  */
 221 struct extern_func {
 222         TAILQ_ENTRY(extern_func) node;
 223         char name[RTE_SWX_NAME_SIZE];
 224         struct struct_type *mailbox_struct_type;
 225         rte_swx_extern_func_t func;
 226         uint32_t struct_id;
 227         uint32_t id;
 228 };
 229
 230 TAILQ_HEAD(extern_func_tailq, extern_func);
 231
 232 struct extern_func_runtime {
 233         uint8_t *mailbox;
 234         rte_swx_extern_func_t func;
 235 };
 236
 237 /*
 238  * Header.
 239  */
 240 struct header {
 241         TAILQ_ENTRY(header) node;
 242         char name[RTE_SWX_NAME_SIZE];
 243         struct struct_type *st;
 244         uint32_t struct_id;
 245         uint32_t id;
 246 };
 247
 248 TAILQ_HEAD(header_tailq, header);
 249
 250 struct header_runtime {
 251         uint8_t *ptr0;
 252 };
 253
 254 struct header_out_runtime {
 255         uint8_t *ptr0;
 256         uint8_t *ptr;
 257         uint32_t n_bytes;
 258 };
 259
 260 /*
 261  * Instruction.
 262  */
 263
 264 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 265  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 266  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 267  * when transferred to packet meta-data and in NBO when transferred to packet
 268  * headers.
 269  */
 270
 271 /* Notation conventions:
 272  *    -Header field: H = h.header.field (dst/src)
 273  *    -Meta-data field: M = m.field (dst/src)
 274  *    -Extern object mailbox field: E = e.field (dst/src)
 275  *    -Extern function mailbox field: F = f.field (dst/src)
 276  *    -Table action data field: T = t.field (src only)
 277  *    -Immediate value: I = 32-bit unsigned value (src only)
 278  */
 279
 280 enum instruction_type {
 281         /* rx m.port_in */
 282         INSTR_RX,
 283
 284         /* tx m.port_out */
 285         INSTR_TX,
 286
 287         /* extract h.header */
 288         INSTR_HDR_EXTRACT,
 289         INSTR_HDR_EXTRACT2,
 290         INSTR_HDR_EXTRACT3,
 291         INSTR_HDR_EXTRACT4,
 292         INSTR_HDR_EXTRACT5,
 293         INSTR_HDR_EXTRACT6,
 294         INSTR_HDR_EXTRACT7,
 295         INSTR_HDR_EXTRACT8,
 296
 297         /* emit h.header */
 298         INSTR_HDR_EMIT,
 299         INSTR_HDR_EMIT_TX,
 300         INSTR_HDR_EMIT2_TX,
 301         INSTR_HDR_EMIT3_TX,
 302         INSTR_HDR_EMIT4_TX,
 303         INSTR_HDR_EMIT5_TX,
 304         INSTR_HDR_EMIT6_TX,
 305         INSTR_HDR_EMIT7_TX,
 306         INSTR_HDR_EMIT8_TX,
 307
 308         /* validate h.header */
 309         INSTR_HDR_VALIDATE,
 310
 311         /* invalidate h.header */
 312         INSTR_HDR_INVALIDATE,
 313
 314         /* mov dst src
 315          * dst = src
 316          * dst = HMEF, src = HMEFTI
 317          */
 318         INSTR_MOV,   /* dst = MEF, src = MEFT */
 319         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 320         INSTR_MOV_I, /* dst = HMEF, src = I */
 321
 322         /* dma h.header t.field
 323          * memcpy(h.header, t.field, sizeof(h.header))
 324          */
 325         INSTR_DMA_HT,
 326         INSTR_DMA_HT2,
 327         INSTR_DMA_HT3,
 328         INSTR_DMA_HT4,
 329         INSTR_DMA_HT5,
 330         INSTR_DMA_HT6,
 331         INSTR_DMA_HT7,
 332         INSTR_DMA_HT8,
 333
 334         /* add dst src
 335          * dst += src
 336          * dst = HMEF, src = HMEFTI
 337          */
 338         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 339         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 340         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 341         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 342         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 343         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 344
 345         /* sub dst src
 346          * dst -= src
 347          * dst = HMEF, src = HMEFTI
 348          */
 349         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 350         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 351         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 352         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 353         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 354         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 355
 356         /* ckadd dst src
 357          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 358          * dst = H, src = {H, h.header}
 359          */
 360         INSTR_ALU_CKADD_FIELD,    /* src = H */
 361         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 362         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 363
 364         /* cksub dst src
 365          * dst = dst '- src
 366          * dst = H, src = H
 367          */
 368         INSTR_ALU_CKSUB_FIELD,
 369
 370         /* and dst src
 371          * dst &= src
 372          * dst = HMEF, src = HMEFTI
 373          */
 374         INSTR_ALU_AND,   /* dst = MEF, src = MEFT */
 375         INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 376         INSTR_ALU_AND_I, /* dst = HMEF, src = I */
 377
 378         /* or dst src
 379          * dst |= src
 380          * dst = HMEF, src = HMEFTI
 381          */
 382         INSTR_ALU_OR,   /* dst = MEF, src = MEFT */
 383         INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 384         INSTR_ALU_OR_I, /* dst = HMEF, src = I */
 385
 386         /* xor dst src
 387          * dst ^= src
 388          * dst = HMEF, src = HMEFTI
 389          */
 390         INSTR_ALU_XOR,   /* dst = MEF, src = MEFT */
 391         INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 392         INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
 393
 394         /* shl dst src
 395          * dst <<= src
 396          * dst = HMEF, src = HMEFTI
 397          */
 398         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 399         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 400         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 401         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 402         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 403         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 404
 405         /* shr dst src
 406          * dst >>= src
 407          * dst = HMEF, src = HMEFTI
 408          */
 409         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 410         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 411         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 412         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 413         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 414         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 415
 416         /* regprefetch REGARRAY index
 417          * prefetch REGARRAY[index]
 418          * index = HMEFTI
 419          */
 420         INSTR_REGPREFETCH_RH, /* index = H */
 421         INSTR_REGPREFETCH_RM, /* index = MEFT */
 422         INSTR_REGPREFETCH_RI, /* index = I */
 423
 424         /* regrd dst REGARRAY index
 425          * dst = REGARRAY[index]
 426          * dst = HMEF, index = HMEFTI
 427          */
 428         INSTR_REGRD_HRH, /* dst = H, index = H */
 429         INSTR_REGRD_HRM, /* dst = H, index = MEFT */
 430         INSTR_REGRD_HRI, /* dst = H, index = I */
 431         INSTR_REGRD_MRH, /* dst = MEF, index = H */
 432         INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
 433         INSTR_REGRD_MRI, /* dst = MEF, index = I */
 434
 435         /* regwr REGARRAY index src
 436          * REGARRAY[index] = src
 437          * index = HMEFTI, src = HMEFTI
 438          */
 439         INSTR_REGWR_RHH, /* index = H, src = H */
 440         INSTR_REGWR_RHM, /* index = H, src = MEFT */
 441         INSTR_REGWR_RHI, /* index = H, src = I */
 442         INSTR_REGWR_RMH, /* index = MEFT, src = H */
 443         INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
 444         INSTR_REGWR_RMI, /* index = MEFT, src = I */
 445         INSTR_REGWR_RIH, /* index = I, src = H */
 446         INSTR_REGWR_RIM, /* index = I, src = MEFT */
 447         INSTR_REGWR_RII, /* index = I, src = I */
 448
 449         /* regadd REGARRAY index src
 450          * REGARRAY[index] += src
 451          * index = HMEFTI, src = HMEFTI
 452          */
 453         INSTR_REGADD_RHH, /* index = H, src = H */
 454         INSTR_REGADD_RHM, /* index = H, src = MEFT */
 455         INSTR_REGADD_RHI, /* index = H, src = I */
 456         INSTR_REGADD_RMH, /* index = MEFT, src = H */
 457         INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
 458         INSTR_REGADD_RMI, /* index = MEFT, src = I */
 459         INSTR_REGADD_RIH, /* index = I, src = H */
 460         INSTR_REGADD_RIM, /* index = I, src = MEFT */
 461         INSTR_REGADD_RII, /* index = I, src = I */
 462
 463         /* metprefetch METARRAY index
 464          * prefetch METARRAY[index]
 465          * index = HMEFTI
 466          */
 467         INSTR_METPREFETCH_H, /* index = H */
 468         INSTR_METPREFETCH_M, /* index = MEFT */
 469         INSTR_METPREFETCH_I, /* index = I */
 470
 471         /* meter METARRAY index length color_in color_out
 472          * color_out = meter(METARRAY[index], length, color_in)
 473          * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
 474          */
 475         INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
 476         INSTR_METER_HHI, /* index = H, length = H, color_in = I */
 477         INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
 478         INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
 479         INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
 480         INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
 481         INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
 482         INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
 483         INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
 484         INSTR_METER_IHI, /* index = I, length = H, color_in = I */
 485         INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
 486         INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
 487
 488         /* table TABLE */
 489         INSTR_TABLE,
 490
 491         /* extern e.obj.func */
 492         INSTR_EXTERN_OBJ,
 493
 494         /* extern f.func */
 495         INSTR_EXTERN_FUNC,
 496
 497         /* jmp LABEL
 498          * Unconditional jump
 499          */
 500         INSTR_JMP,
 501
 502         /* jmpv LABEL h.header
 503          * Jump if header is valid
 504          */
 505         INSTR_JMP_VALID,
 506
 507         /* jmpnv LABEL h.header
 508          * Jump if header is invalid
 509          */
 510         INSTR_JMP_INVALID,
 511
 512         /* jmph LABEL
 513          * Jump if table lookup hit
 514          */
 515         INSTR_JMP_HIT,
 516
 517         /* jmpnh LABEL
 518          * Jump if table lookup miss
 519          */
 520         INSTR_JMP_MISS,
 521
 522         /* jmpa LABEL ACTION
 523          * Jump if action run
 524          */
 525         INSTR_JMP_ACTION_HIT,
 526
 527         /* jmpna LABEL ACTION
 528          * Jump if action not run
 529          */
 530         INSTR_JMP_ACTION_MISS,
 531
 532         /* jmpeq LABEL a b
 533          * Jump is a is equal to b
 534          * a = HMEFT, b = HMEFTI
 535          */
 536         INSTR_JMP_EQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
 537         INSTR_JMP_EQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
 538         INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 539
 540         /* jmpneq LABEL a b
 541          * Jump is a is not equal to b
 542          * a = HMEFT, b = HMEFTI
 543          */
 544         INSTR_JMP_NEQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
 545         INSTR_JMP_NEQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
 546         INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 547
 548         /* jmplt LABEL a b
 549          * Jump if a is less than b
 550          * a = HMEFT, b = HMEFTI
 551          */
 552         INSTR_JMP_LT,    /* a = MEF, b = MEF */
 553         INSTR_JMP_LT_MH, /* a = MEF, b = H */
 554         INSTR_JMP_LT_HM, /* a = H, b = MEF */
 555         INSTR_JMP_LT_HH, /* a = H, b = H */
 556         INSTR_JMP_LT_MI, /* a = MEF, b = I */
 557         INSTR_JMP_LT_HI, /* a = H, b = I */
 558
 559         /* jmpgt LABEL a b
 560          * Jump if a is greater than b
 561          * a = HMEFT, b = HMEFTI
 562          */
 563         INSTR_JMP_GT,    /* a = MEF, b = MEF */
 564         INSTR_JMP_GT_MH, /* a = MEF, b = H */
 565         INSTR_JMP_GT_HM, /* a = H, b = MEF */
 566         INSTR_JMP_GT_HH, /* a = H, b = H */
 567         INSTR_JMP_GT_MI, /* a = MEF, b = I */
 568         INSTR_JMP_GT_HI, /* a = H, b = I */
 569
 570         /* return
 571          * Return from action
 572          */
 573         INSTR_RETURN,
 574 };
 575
 576 struct instr_operand {
 577         uint8_t struct_id;
 578         uint8_t n_bits;
 579         uint8_t offset;
 580         uint8_t pad;
 581 };
 582
 583 struct instr_io {
 584         struct {
 585                 uint8_t offset;
 586                 uint8_t n_bits;
 587                 uint8_t pad[2];
 588         } io;
 589
 590         struct {
 591                 uint8_t header_id[8];
 592                 uint8_t struct_id[8];
 593                 uint8_t n_bytes[8];
 594         } hdr;
 595 };
 596
 597 struct instr_hdr_validity {
 598         uint8_t header_id;
 599 };
 600
 601 struct instr_table {
 602         uint8_t table_id;
 603 };
 604
 605 struct instr_extern_obj {
 606         uint8_t ext_obj_id;
 607         uint8_t func_id;
 608 };
 609
 610 struct instr_extern_func {
 611         uint8_t ext_func_id;
 612 };
 613
 614 struct instr_dst_src {
 615         struct instr_operand dst;
 616         union {
 617                 struct instr_operand src;
 618                 uint64_t src_val;
 619         };
 620 };
 621
 622 struct instr_regarray {
 623         uint8_t regarray_id;
 624         uint8_t pad[3];
 625
 626         union {
 627                 struct instr_operand idx;
 628                 uint32_t idx_val;
 629         };
 630
 631         union {
 632                 struct instr_operand dstsrc;
 633                 uint64_t dstsrc_val;
 634         };
 635 };
 636
 637 struct instr_meter {
 638         uint8_t metarray_id;
 639         uint8_t pad[3];
 640
 641         union {
 642                 struct instr_operand idx;
 643                 uint32_t idx_val;
 644         };
 645
 646         struct instr_operand length;
 647
 648         union {
 649                 struct instr_operand color_in;
 650                 uint32_t color_in_val;
 651         };
 652
 653         struct instr_operand color_out;
 654 };
 655
 656 struct instr_dma {
 657         struct {
 658                 uint8_t header_id[8];
 659                 uint8_t struct_id[8];
 660         } dst;
 661
 662         struct {
 663                 uint8_t offset[8];
 664         } src;
 665
 666         uint16_t n_bytes[8];
 667 };
 668
 669 struct instr_jmp {
 670         struct instruction *ip;
 671
 672         union {
 673                 struct instr_operand a;
 674                 uint8_t header_id;
 675                 uint8_t action_id;
 676         };
 677
 678         union {
 679                 struct instr_operand b;
 680                 uint64_t b_val;
 681         };
 682 };
 683
 684 struct instruction {
 685         enum instruction_type type;
 686         union {
 687                 struct instr_io io;
 688                 struct instr_hdr_validity valid;
 689                 struct instr_dst_src mov;
 690                 struct instr_regarray regarray;
 691                 struct instr_meter meter;
 692                 struct instr_dma dma;
 693                 struct instr_dst_src alu;
 694                 struct instr_table table;
 695                 struct instr_extern_obj ext_obj;
 696                 struct instr_extern_func ext_func;
 697                 struct instr_jmp jmp;
 698         };
 699 };
 700
 701 struct instruction_data {
 702         char label[RTE_SWX_NAME_SIZE];
 703         char jmp_label[RTE_SWX_NAME_SIZE];
 704         uint32_t n_users; /* user = jmp instruction to this instruction. */
 705         int invalid;
 706 };
 707
 708 /*
 709  * Action.
 710  */
 711 struct action {
 712         TAILQ_ENTRY(action) node;
 713         char name[RTE_SWX_NAME_SIZE];
 714         struct struct_type *st;
 715         struct instruction *instructions;
 716         uint32_t n_instructions;
 717         uint32_t id;
 718 };
 719
 720 TAILQ_HEAD(action_tailq, action);
 721
 722 /*
 723  * Table.
 724  */
 725 struct table_type {
 726         TAILQ_ENTRY(table_type) node;
 727         char name[RTE_SWX_NAME_SIZE];
 728         enum rte_swx_table_match_type match_type;
 729         struct rte_swx_table_ops ops;
 730 };
 731
 732 TAILQ_HEAD(table_type_tailq, table_type);
 733
 734 struct match_field {
 735         enum rte_swx_table_match_type match_type;
 736         struct field *field;
 737 };
 738
 739 struct table {
 740         TAILQ_ENTRY(table) node;
 741         char name[RTE_SWX_NAME_SIZE];
 742         char args[RTE_SWX_NAME_SIZE];
 743         struct table_type *type; /* NULL when n_fields == 0. */
 744
 745         /* Match. */
 746         struct match_field *fields;
 747         uint32_t n_fields;
 748         int is_header; /* Only valid when n_fields > 0. */
 749         struct header *header; /* Only valid when n_fields > 0. */
 750
 751         /* Action. */
 752         struct action **actions;
 753         struct action *default_action;
 754         uint8_t *default_action_data;
 755         uint32_t n_actions;
 756         int default_action_is_const;
 757         uint32_t action_data_size_max;
 758
 759         uint32_t size;
 760         uint32_t id;
 761 };
 762
 763 TAILQ_HEAD(table_tailq, table);
 764
 765 struct table_runtime {
 766         rte_swx_table_lookup_t func;
 767         void *mailbox;
 768         uint8_t **key;
 769 };
 770
 771 /*
 772  * Register array.
 773  */
 774 struct regarray {
 775         TAILQ_ENTRY(regarray) node;
 776         char name[RTE_SWX_NAME_SIZE];
 777         uint64_t init_val;
 778         uint32_t size;
 779         uint32_t id;
 780 };
 781
 782 TAILQ_HEAD(regarray_tailq, regarray);
 783
 784 struct regarray_runtime {
 785         uint64_t *regarray;
 786         uint32_t size_mask;
 787 };
 788
 789 /*
 790  * Meter array.
 791  */
 792 struct meter_profile {
 793         TAILQ_ENTRY(meter_profile) node;
 794         char name[RTE_SWX_NAME_SIZE];
 795         struct rte_meter_trtcm_params params;
 796         struct rte_meter_trtcm_profile profile;
 797         uint32_t n_users;
 798 };
 799
 800 TAILQ_HEAD(meter_profile_tailq, meter_profile);
 801
 802 struct metarray {
 803         TAILQ_ENTRY(metarray) node;
 804         char name[RTE_SWX_NAME_SIZE];
 805         uint32_t size;
 806         uint32_t id;
 807 };
 808
 809 TAILQ_HEAD(metarray_tailq, metarray);
 810
 811 struct meter {
 812         struct rte_meter_trtcm m;
 813         struct meter_profile *profile;
 814         enum rte_color color_mask;
 815         uint8_t pad[20];
 816
 817         uint64_t n_pkts[RTE_COLORS];
 818         uint64_t n_bytes[RTE_COLORS];
 819 };
 820
 821 struct metarray_runtime {
 822         struct meter *metarray;
 823         uint32_t size_mask;
 824 };
 825
 826 /*
 827  * Pipeline.
 828  */
 829 struct thread {
 830         /* Packet. */
 831         struct rte_swx_pkt pkt;
 832         uint8_t *ptr;
 833
 834         /* Structures. */
 835         uint8_t **structs;
 836
 837         /* Packet headers. */
 838         struct header_runtime *headers; /* Extracted or generated headers. */
 839         struct header_out_runtime *headers_out; /* Emitted headers. */
 840         uint8_t *header_storage;
 841         uint8_t *header_out_storage;
 842         uint64_t valid_headers;
 843         uint32_t n_headers_out;
 844
 845         /* Packet meta-data. */
 846         uint8_t *metadata;
 847
 848         /* Tables. */
 849         struct table_runtime *tables;
 850         struct rte_swx_table_state *table_state;
 851         uint64_t action_id;
 852         int hit; /* 0 = Miss, 1 = Hit. */
 853
 854         /* Extern objects and functions. */
 855         struct extern_obj_runtime *extern_objs;
 856         struct extern_func_runtime *extern_funcs;
 857
 858         /* Instructions. */
 859         struct instruction *ip;
 860         struct instruction *ret;
 861 };
 862
 863 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 864 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 865 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 866
 867 #define HEADER_VALID(thread, header_id) \
 868         MASK64_BIT_GET((thread)->valid_headers, header_id)
 869
 870 #define ALU(thread, ip, operator)  \
 871 {                                                                              \
 872         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 873         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 874         uint64_t dst64 = *dst64_ptr;                                           \
 875         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 876         uint64_t dst = dst64 & dst64_mask;                                     \
 877                                                                                \
 878         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 879         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 880         uint64_t src64 = *src64_ptr;                                           \
 881         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 882         uint64_t src = src64 & src64_mask;                                     \
 883                                                                                \
 884         uint64_t result = dst operator src;                                    \
 885                                                                                \
 886         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 887 }
 888
 889 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 890
 891 #define ALU_S(thread, ip, operator)  \
 892 {                                                                              \
 893         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 894         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 895         uint64_t dst64 = *dst64_ptr;                                           \
 896         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 897         uint64_t dst = dst64 & dst64_mask;                                     \
 898                                                                                \
 899         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 900         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 901         uint64_t src64 = *src64_ptr;                                           \
 902         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 903                                                                                \
 904         uint64_t result = dst operator src;                                    \
 905                                                                                \
 906         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 907 }
 908
 909 #define ALU_MH ALU_S
 910
 911 #define ALU_HM(thread, ip, operator)  \
 912 {                                                                              \
 913         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 914         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 915         uint64_t dst64 = *dst64_ptr;                                           \
 916         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 917         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 918                                                                                \
 919         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 920         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 921         uint64_t src64 = *src64_ptr;                                           \
 922         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 923         uint64_t src = src64 & src64_mask;                                     \
 924                                                                                \
 925         uint64_t result = dst operator src;                                    \
 926         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 927                                                                                \
 928         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 929 }
 930
 931 #define ALU_HH(thread, ip, operator)  \
 932 {                                                                              \
 933         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 934         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 935         uint64_t dst64 = *dst64_ptr;                                           \
 936         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 937         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 938                                                                                \
 939         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 940         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 941         uint64_t src64 = *src64_ptr;                                           \
 942         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 943                                                                                \
 944         uint64_t result = dst operator src;                                    \
 945         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 946                                                                                \
 947         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 948 }
 949
 950 #else
 951
 952 #define ALU_S ALU
 953 #define ALU_MH ALU
 954 #define ALU_HM ALU
 955 #define ALU_HH ALU
 956
 957 #endif
 958
 959 #define ALU_I(thread, ip, operator)  \
 960 {                                                                              \
 961         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 962         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 963         uint64_t dst64 = *dst64_ptr;                                           \
 964         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 965         uint64_t dst = dst64 & dst64_mask;                                     \
 966                                                                                \
 967         uint64_t src = (ip)->alu.src_val;                                      \
 968                                                                                \
 969         uint64_t result = dst operator src;                                    \
 970                                                                                \
 971         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 972 }
 973
 974 #define ALU_MI ALU_I
 975
 976 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 977
 978 #define ALU_HI(thread, ip, operator)  \
 979 {                                                                              \
 980         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 981         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 982         uint64_t dst64 = *dst64_ptr;                                           \
 983         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 984         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 985                                                                                \
 986         uint64_t src = (ip)->alu.src_val;                                      \
 987                                                                                \
 988         uint64_t result = dst operator src;                                    \
 989         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 990                                                                                \
 991         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 992 }
 993
 994 #else
 995
 996 #define ALU_HI ALU_I
 997
 998 #endif
 999
1000 #define MOV(thread, ip)  \
1001 {                                                                              \
1002         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1003         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1004         uint64_t dst64 = *dst64_ptr;                                           \
1005         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1006                                                                                \
1007         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1008         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1009         uint64_t src64 = *src64_ptr;                                           \
1010         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1011         uint64_t src = src64 & src64_mask;                                     \
1012                                                                                \
1013         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1014 }
1015
1016 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1017
1018 #define MOV_S(thread, ip)  \
1019 {                                                                              \
1020         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1021         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1022         uint64_t dst64 = *dst64_ptr;                                           \
1023         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1024                                                                                \
1025         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1026         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1027         uint64_t src64 = *src64_ptr;                                           \
1028         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
1029                                                                                \
1030         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1031 }
1032
1033 #else
1034
1035 #define MOV_S MOV
1036
1037 #endif
1038
1039 #define MOV_I(thread, ip)  \
1040 {                                                                              \
1041         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1042         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1043         uint64_t dst64 = *dst64_ptr;                                           \
1044         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1045                                                                                \
1046         uint64_t src = (ip)->mov.src_val;                                      \
1047                                                                                \
1048         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1049 }
1050
1051 #define JMP_CMP(thread, ip, operator)  \
1052 {                                                                              \
1053         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1054         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1055         uint64_t a64 = *a64_ptr;                                               \
1056         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1057         uint64_t a = a64 & a64_mask;                                           \
1058                                                                                \
1059         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1060         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1061         uint64_t b64 = *b64_ptr;                                               \
1062         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1063         uint64_t b = b64 & b64_mask;                                           \
1064                                                                                \
1065         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1066 }
1067
1068 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1069
1070 #define JMP_CMP_S(thread, ip, operator)  \
1071 {                                                                              \
1072         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1073         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1074         uint64_t a64 = *a64_ptr;                                               \
1075         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1076         uint64_t a = a64 & a64_mask;                                           \
1077                                                                                \
1078         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1079         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1080         uint64_t b64 = *b64_ptr;                                               \
1081         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1082                                                                                \
1083         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1084 }
1085
1086 #define JMP_CMP_MH JMP_CMP_S
1087
1088 #define JMP_CMP_HM(thread, ip, operator)  \
1089 {                                                                              \
1090         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1091         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1092         uint64_t a64 = *a64_ptr;                                               \
1093         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1094                                                                                \
1095         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1096         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1097         uint64_t b64 = *b64_ptr;                                               \
1098         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1099         uint64_t b = b64 & b64_mask;                                           \
1100                                                                                \
1101         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1102 }
1103
1104 #define JMP_CMP_HH(thread, ip, operator)  \
1105 {                                                                              \
1106         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1107         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1108         uint64_t a64 = *a64_ptr;                                               \
1109         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1110                                                                                \
1111         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1112         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1113         uint64_t b64 = *b64_ptr;                                               \
1114         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1115                                                                                \
1116         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1117 }
1118
1119 #else
1120
1121 #define JMP_CMP_S JMP_CMP
1122 #define JMP_CMP_MH JMP_CMP
1123 #define JMP_CMP_HM JMP_CMP
1124 #define JMP_CMP_HH JMP_CMP
1125
1126 #endif
1127
1128 #define JMP_CMP_I(thread, ip, operator)  \
1129 {                                                                              \
1130         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1131         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1132         uint64_t a64 = *a64_ptr;                                               \
1133         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1134         uint64_t a = a64 & a64_mask;                                           \
1135                                                                                \
1136         uint64_t b = (ip)->jmp.b_val;                                          \
1137                                                                                \
1138         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1139 }
1140
1141 #define JMP_CMP_MI JMP_CMP_I
1142
1143 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1144
1145 #define JMP_CMP_HI(thread, ip, operator)  \
1146 {                                                                              \
1147         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1148         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1149         uint64_t a64 = *a64_ptr;                                               \
1150         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1151                                                                                \
1152         uint64_t b = (ip)->jmp.b_val;                                          \
1153                                                                                \
1154         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1155 }
1156
1157 #else
1158
1159 #define JMP_CMP_HI JMP_CMP_I
1160
1161 #endif
1162
1163 #define METADATA_READ(thread, offset, n_bits)                                  \
1164 ({                                                                             \
1165         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1166         uint64_t m64 = *m64_ptr;                                               \
1167         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1168         (m64 & m64_mask);                                                      \
1169 })
1170
1171 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
1172 {                                                                              \
1173         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1174         uint64_t m64 = *m64_ptr;                                               \
1175         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1176                                                                                \
1177         uint64_t m_new = value;                                                \
1178                                                                                \
1179         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
1180 }
1181
1182 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1183 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1184 #endif
1185
1186 struct rte_swx_pipeline {
1187         struct struct_type_tailq struct_types;
1188         struct port_in_type_tailq port_in_types;
1189         struct port_in_tailq ports_in;
1190         struct port_out_type_tailq port_out_types;
1191         struct port_out_tailq ports_out;
1192         struct extern_type_tailq extern_types;
1193         struct extern_obj_tailq extern_objs;
1194         struct extern_func_tailq extern_funcs;
1195         struct header_tailq headers;
1196         struct struct_type *metadata_st;
1197         uint32_t metadata_struct_id;
1198         struct action_tailq actions;
1199         struct table_type_tailq table_types;
1200         struct table_tailq tables;
1201         struct regarray_tailq regarrays;
1202         struct meter_profile_tailq meter_profiles;
1203         struct metarray_tailq metarrays;
1204
1205         struct port_in_runtime *in;
1206         struct port_out_runtime *out;
1207         struct instruction **action_instructions;
1208         struct rte_swx_table_state *table_state;
1209         struct regarray_runtime *regarray_runtime;
1210         struct metarray_runtime *metarray_runtime;
1211         struct instruction *instructions;
1212         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1213
1214         uint32_t n_structs;
1215         uint32_t n_ports_in;
1216         uint32_t n_ports_out;
1217         uint32_t n_extern_objs;
1218         uint32_t n_extern_funcs;
1219         uint32_t n_actions;
1220         uint32_t n_tables;
1221         uint32_t n_regarrays;
1222         uint32_t n_metarrays;
1223         uint32_t n_headers;
1224         uint32_t thread_id;
1225         uint32_t port_id;
1226         uint32_t n_instructions;
1227         int build_done;
1228         int numa_node;
1229 };
1230
1231 /*
1232  * Struct.
1233  */
1234 static struct struct_type *
1235 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1236 {
1237         struct struct_type *elem;
1238
1239         TAILQ_FOREACH(elem, &p->struct_types, node)
1240                 if (strcmp(elem->name, name) == 0)
1241                         return elem;
1242
1243         return NULL;
1244 }
1245
1246 static struct field *
1247 struct_type_field_find(struct struct_type *st, const char *name)
1248 {
1249         uint32_t i;
1250
1251         for (i = 0; i < st->n_fields; i++) {
1252                 struct field *f = &st->fields[i];
1253
1254                 if (strcmp(f->name, name) == 0)
1255                         return f;
1256         }
1257
1258         return NULL;
1259 }
1260
1261 int
1262 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1263                                       const char *name,
1264                                       struct rte_swx_field_params *fields,
1265                                       uint32_t n_fields)
1266 {
1267         struct struct_type *st;
1268         uint32_t i;
1269
1270         CHECK(p, EINVAL);
1271         CHECK_NAME(name, EINVAL);
1272         CHECK(fields, EINVAL);
1273         CHECK(n_fields, EINVAL);
1274
1275         for (i = 0; i < n_fields; i++) {
1276                 struct rte_swx_field_params *f = &fields[i];
1277                 uint32_t j;
1278
1279                 CHECK_NAME(f->name, EINVAL);
1280                 CHECK(f->n_bits, EINVAL);
1281                 CHECK(f->n_bits <= 64, EINVAL);
1282                 CHECK((f->n_bits & 7) == 0, EINVAL);
1283
1284                 for (j = 0; j < i; j++) {
1285                         struct rte_swx_field_params *f_prev = &fields[j];
1286
1287                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
1288                 }
1289         }
1290
1291         CHECK(!struct_type_find(p, name), EEXIST);
1292
1293         /* Node allocation. */
1294         st = calloc(1, sizeof(struct struct_type));
1295         CHECK(st, ENOMEM);
1296
1297         st->fields = calloc(n_fields, sizeof(struct field));
1298         if (!st->fields) {
1299                 free(st);
1300                 CHECK(0, ENOMEM);
1301         }
1302
1303         /* Node initialization. */
1304         strcpy(st->name, name);
1305         for (i = 0; i < n_fields; i++) {
1306                 struct field *dst = &st->fields[i];
1307                 struct rte_swx_field_params *src = &fields[i];
1308
1309                 strcpy(dst->name, src->name);
1310                 dst->n_bits = src->n_bits;
1311                 dst->offset = st->n_bits;
1312
1313                 st->n_bits += src->n_bits;
1314         }
1315         st->n_fields = n_fields;
1316
1317         /* Node add to tailq. */
1318         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1319
1320         return 0;
1321 }
1322
1323 static int
1324 struct_build(struct rte_swx_pipeline *p)
1325 {
1326         uint32_t i;
1327
1328         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1329                 struct thread *t = &p->threads[i];
1330
1331                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1332                 CHECK(t->structs, ENOMEM);
1333         }
1334
1335         return 0;
1336 }
1337
1338 static void
1339 struct_build_free(struct rte_swx_pipeline *p)
1340 {
1341         uint32_t i;
1342
1343         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1344                 struct thread *t = &p->threads[i];
1345
1346                 free(t->structs);
1347                 t->structs = NULL;
1348         }
1349 }
1350
1351 static void
1352 struct_free(struct rte_swx_pipeline *p)
1353 {
1354         struct_build_free(p);
1355
1356         /* Struct types. */
1357         for ( ; ; ) {
1358                 struct struct_type *elem;
1359
1360                 elem = TAILQ_FIRST(&p->struct_types);
1361                 if (!elem)
1362                         break;
1363
1364                 TAILQ_REMOVE(&p->struct_types, elem, node);
1365                 free(elem->fields);
1366                 free(elem);
1367         }
1368 }
1369
1370 /*
1371  * Input port.
1372  */
1373 static struct port_in_type *
1374 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1375 {
1376         struct port_in_type *elem;
1377
1378         if (!name)
1379                 return NULL;
1380
1381         TAILQ_FOREACH(elem, &p->port_in_types, node)
1382                 if (strcmp(elem->name, name) == 0)
1383                         return elem;
1384
1385         return NULL;
1386 }
1387
1388 int
1389 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1390                                        const char *name,
1391                                        struct rte_swx_port_in_ops *ops)
1392 {
1393         struct port_in_type *elem;
1394
1395         CHECK(p, EINVAL);
1396         CHECK_NAME(name, EINVAL);
1397         CHECK(ops, EINVAL);
1398         CHECK(ops->create, EINVAL);
1399         CHECK(ops->free, EINVAL);
1400         CHECK(ops->pkt_rx, EINVAL);
1401         CHECK(ops->stats_read, EINVAL);
1402
1403         CHECK(!port_in_type_find(p, name), EEXIST);
1404
1405         /* Node allocation. */
1406         elem = calloc(1, sizeof(struct port_in_type));
1407         CHECK(elem, ENOMEM);
1408
1409         /* Node initialization. */
1410         strcpy(elem->name, name);
1411         memcpy(&elem->ops, ops, sizeof(*ops));
1412
1413         /* Node add to tailq. */
1414         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1415
1416         return 0;
1417 }
1418
1419 static struct port_in *
1420 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1421 {
1422         struct port_in *port;
1423
1424         TAILQ_FOREACH(port, &p->ports_in, node)
1425                 if (port->id == port_id)
1426                         return port;
1427
1428         return NULL;
1429 }
1430
1431 int
1432 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1433                                 uint32_t port_id,
1434                                 const char *port_type_name,
1435                                 void *args)
1436 {
1437         struct port_in_type *type = NULL;
1438         struct port_in *port = NULL;
1439         void *obj = NULL;
1440
1441         CHECK(p, EINVAL);
1442
1443         CHECK(!port_in_find(p, port_id), EINVAL);
1444
1445         CHECK_NAME(port_type_name, EINVAL);
1446         type = port_in_type_find(p, port_type_name);
1447         CHECK(type, EINVAL);
1448
1449         obj = type->ops.create(args);
1450         CHECK(obj, ENODEV);
1451
1452         /* Node allocation. */
1453         port = calloc(1, sizeof(struct port_in));
1454         CHECK(port, ENOMEM);
1455
1456         /* Node initialization. */
1457         port->type = type;
1458         port->obj = obj;
1459         port->id = port_id;
1460
1461         /* Node add to tailq. */
1462         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1463         if (p->n_ports_in < port_id + 1)
1464                 p->n_ports_in = port_id + 1;
1465
1466         return 0;
1467 }
1468
1469 static int
1470 port_in_build(struct rte_swx_pipeline *p)
1471 {
1472         struct port_in *port;
1473         uint32_t i;
1474
1475         CHECK(p->n_ports_in, EINVAL);
1476         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1477
1478         for (i = 0; i < p->n_ports_in; i++)
1479                 CHECK(port_in_find(p, i), EINVAL);
1480
1481         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1482         CHECK(p->in, ENOMEM);
1483
1484         TAILQ_FOREACH(port, &p->ports_in, node) {
1485                 struct port_in_runtime *in = &p->in[port->id];
1486
1487                 in->pkt_rx = port->type->ops.pkt_rx;
1488                 in->obj = port->obj;
1489         }
1490
1491         return 0;
1492 }
1493
1494 static void
1495 port_in_build_free(struct rte_swx_pipeline *p)
1496 {
1497         free(p->in);
1498         p->in = NULL;
1499 }
1500
1501 static void
1502 port_in_free(struct rte_swx_pipeline *p)
1503 {
1504         port_in_build_free(p);
1505
1506         /* Input ports. */
1507         for ( ; ; ) {
1508                 struct port_in *port;
1509
1510                 port = TAILQ_FIRST(&p->ports_in);
1511                 if (!port)
1512                         break;
1513
1514                 TAILQ_REMOVE(&p->ports_in, port, node);
1515                 port->type->ops.free(port->obj);
1516                 free(port);
1517         }
1518
1519         /* Input port types. */
1520         for ( ; ; ) {
1521                 struct port_in_type *elem;
1522
1523                 elem = TAILQ_FIRST(&p->port_in_types);
1524                 if (!elem)
1525                         break;
1526
1527                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1528                 free(elem);
1529         }
1530 }
1531
1532 /*
1533  * Output port.
1534  */
1535 static struct port_out_type *
1536 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1537 {
1538         struct port_out_type *elem;
1539
1540         if (!name)
1541                 return NULL;
1542
1543         TAILQ_FOREACH(elem, &p->port_out_types, node)
1544                 if (!strcmp(elem->name, name))
1545                         return elem;
1546
1547         return NULL;
1548 }
1549
1550 int
1551 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1552                                         const char *name,
1553                                         struct rte_swx_port_out_ops *ops)
1554 {
1555         struct port_out_type *elem;
1556
1557         CHECK(p, EINVAL);
1558         CHECK_NAME(name, EINVAL);
1559         CHECK(ops, EINVAL);
1560         CHECK(ops->create, EINVAL);
1561         CHECK(ops->free, EINVAL);
1562         CHECK(ops->pkt_tx, EINVAL);
1563         CHECK(ops->stats_read, EINVAL);
1564
1565         CHECK(!port_out_type_find(p, name), EEXIST);
1566
1567         /* Node allocation. */
1568         elem = calloc(1, sizeof(struct port_out_type));
1569         CHECK(elem, ENOMEM);
1570
1571         /* Node initialization. */
1572         strcpy(elem->name, name);
1573         memcpy(&elem->ops, ops, sizeof(*ops));
1574
1575         /* Node add to tailq. */
1576         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1577
1578         return 0;
1579 }
1580
1581 static struct port_out *
1582 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1583 {
1584         struct port_out *port;
1585
1586         TAILQ_FOREACH(port, &p->ports_out, node)
1587                 if (port->id == port_id)
1588                         return port;
1589
1590         return NULL;
1591 }
1592
1593 int
1594 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1595                                  uint32_t port_id,
1596                                  const char *port_type_name,
1597                                  void *args)
1598 {
1599         struct port_out_type *type = NULL;
1600         struct port_out *port = NULL;
1601         void *obj = NULL;
1602
1603         CHECK(p, EINVAL);
1604
1605         CHECK(!port_out_find(p, port_id), EINVAL);
1606
1607         CHECK_NAME(port_type_name, EINVAL);
1608         type = port_out_type_find(p, port_type_name);
1609         CHECK(type, EINVAL);
1610
1611         obj = type->ops.create(args);
1612         CHECK(obj, ENODEV);
1613
1614         /* Node allocation. */
1615         port = calloc(1, sizeof(struct port_out));
1616         CHECK(port, ENOMEM);
1617
1618         /* Node initialization. */
1619         port->type = type;
1620         port->obj = obj;
1621         port->id = port_id;
1622
1623         /* Node add to tailq. */
1624         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1625         if (p->n_ports_out < port_id + 1)
1626                 p->n_ports_out = port_id + 1;
1627
1628         return 0;
1629 }
1630
1631 static int
1632 port_out_build(struct rte_swx_pipeline *p)
1633 {
1634         struct port_out *port;
1635         uint32_t i;
1636
1637         CHECK(p->n_ports_out, EINVAL);
1638
1639         for (i = 0; i < p->n_ports_out; i++)
1640                 CHECK(port_out_find(p, i), EINVAL);
1641
1642         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1643         CHECK(p->out, ENOMEM);
1644
1645         TAILQ_FOREACH(port, &p->ports_out, node) {
1646                 struct port_out_runtime *out = &p->out[port->id];
1647
1648                 out->pkt_tx = port->type->ops.pkt_tx;
1649                 out->flush = port->type->ops.flush;
1650                 out->obj = port->obj;
1651         }
1652
1653         return 0;
1654 }
1655
1656 static void
1657 port_out_build_free(struct rte_swx_pipeline *p)
1658 {
1659         free(p->out);
1660         p->out = NULL;
1661 }
1662
1663 static void
1664 port_out_free(struct rte_swx_pipeline *p)
1665 {
1666         port_out_build_free(p);
1667
1668         /* Output ports. */
1669         for ( ; ; ) {
1670                 struct port_out *port;
1671
1672                 port = TAILQ_FIRST(&p->ports_out);
1673                 if (!port)
1674                         break;
1675
1676                 TAILQ_REMOVE(&p->ports_out, port, node);
1677                 port->type->ops.free(port->obj);
1678                 free(port);
1679         }
1680
1681         /* Output port types. */
1682         for ( ; ; ) {
1683                 struct port_out_type *elem;
1684
1685                 elem = TAILQ_FIRST(&p->port_out_types);
1686                 if (!elem)
1687                         break;
1688
1689                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1690                 free(elem);
1691         }
1692 }
1693
1694 /*
1695  * Extern object.
1696  */
1697 static struct extern_type *
1698 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1699 {
1700         struct extern_type *elem;
1701
1702         TAILQ_FOREACH(elem, &p->extern_types, node)
1703                 if (strcmp(elem->name, name) == 0)
1704                         return elem;
1705
1706         return NULL;
1707 }
1708
1709 static struct extern_type_member_func *
1710 extern_type_member_func_find(struct extern_type *type, const char *name)
1711 {
1712         struct extern_type_member_func *elem;
1713
1714         TAILQ_FOREACH(elem, &type->funcs, node)
1715                 if (strcmp(elem->name, name) == 0)
1716                         return elem;
1717
1718         return NULL;
1719 }
1720
1721 static struct extern_obj *
1722 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1723 {
1724         struct extern_obj *elem;
1725
1726         TAILQ_FOREACH(elem, &p->extern_objs, node)
1727                 if (strcmp(elem->name, name) == 0)
1728                         return elem;
1729
1730         return NULL;
1731 }
1732
1733 static struct extern_type_member_func *
1734 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1735                              const char *name,
1736                              struct extern_obj **obj)
1737 {
1738         struct extern_obj *object;
1739         struct extern_type_member_func *func;
1740         char *object_name, *func_name;
1741
1742         if (name[0] != 'e' || name[1] != '.')
1743                 return NULL;
1744
1745         object_name = strdup(&name[2]);
1746         if (!object_name)
1747                 return NULL;
1748
1749         func_name = strchr(object_name, '.');
1750         if (!func_name) {
1751                 free(object_name);
1752                 return NULL;
1753         }
1754
1755         *func_name = 0;
1756         func_name++;
1757
1758         object = extern_obj_find(p, object_name);
1759         if (!object) {
1760                 free(object_name);
1761                 return NULL;
1762         }
1763
1764         func = extern_type_member_func_find(object->type, func_name);
1765         if (!func) {
1766                 free(object_name);
1767                 return NULL;
1768         }
1769
1770         if (obj)
1771                 *obj = object;
1772
1773         free(object_name);
1774         return func;
1775 }
1776
1777 static struct field *
1778 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1779                                const char *name,
1780                                struct extern_obj **object)
1781 {
1782         struct extern_obj *obj;
1783         struct field *f;
1784         char *obj_name, *field_name;
1785
1786         if ((name[0] != 'e') || (name[1] != '.'))
1787                 return NULL;
1788
1789         obj_name = strdup(&name[2]);
1790         if (!obj_name)
1791                 return NULL;
1792
1793         field_name = strchr(obj_name, '.');
1794         if (!field_name) {
1795                 free(obj_name);
1796                 return NULL;
1797         }
1798
1799         *field_name = 0;
1800         field_name++;
1801
1802         obj = extern_obj_find(p, obj_name);
1803         if (!obj) {
1804                 free(obj_name);
1805                 return NULL;
1806         }
1807
1808         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1809         if (!f) {
1810                 free(obj_name);
1811                 return NULL;
1812         }
1813
1814         if (object)
1815                 *object = obj;
1816
1817         free(obj_name);
1818         return f;
1819 }
1820
1821 int
1822 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1823         const char *name,
1824         const char *mailbox_struct_type_name,
1825         rte_swx_extern_type_constructor_t constructor,
1826         rte_swx_extern_type_destructor_t destructor)
1827 {
1828         struct extern_type *elem;
1829         struct struct_type *mailbox_struct_type;
1830
1831         CHECK(p, EINVAL);
1832
1833         CHECK_NAME(name, EINVAL);
1834         CHECK(!extern_type_find(p, name), EEXIST);
1835
1836         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1837         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1838         CHECK(mailbox_struct_type, EINVAL);
1839
1840         CHECK(constructor, EINVAL);
1841         CHECK(destructor, EINVAL);
1842
1843         /* Node allocation. */
1844         elem = calloc(1, sizeof(struct extern_type));
1845         CHECK(elem, ENOMEM);
1846
1847         /* Node initialization. */
1848         strcpy(elem->name, name);
1849         elem->mailbox_struct_type = mailbox_struct_type;
1850         elem->constructor = constructor;
1851         elem->destructor = destructor;
1852         TAILQ_INIT(&elem->funcs);
1853
1854         /* Node add to tailq. */
1855         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1856
1857         return 0;
1858 }
1859
1860 int
1861 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1862         const char *extern_type_name,
1863         const char *name,
1864         rte_swx_extern_type_member_func_t member_func)
1865 {
1866         struct extern_type *type;
1867         struct extern_type_member_func *type_member;
1868
1869         CHECK(p, EINVAL);
1870
1871         CHECK_NAME(extern_type_name, EINVAL);
1872         type = extern_type_find(p, extern_type_name);
1873         CHECK(type, EINVAL);
1874         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1875
1876         CHECK_NAME(name, EINVAL);
1877         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1878
1879         CHECK(member_func, EINVAL);
1880
1881         /* Node allocation. */
1882         type_member = calloc(1, sizeof(struct extern_type_member_func));
1883         CHECK(type_member, ENOMEM);
1884
1885         /* Node initialization. */
1886         strcpy(type_member->name, name);
1887         type_member->func = member_func;
1888         type_member->id = type->n_funcs;
1889
1890         /* Node add to tailq. */
1891         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1892         type->n_funcs++;
1893
1894         return 0;
1895 }
1896
1897 int
1898 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1899                                       const char *extern_type_name,
1900                                       const char *name,
1901                                       const char *args)
1902 {
1903         struct extern_type *type;
1904         struct extern_obj *obj;
1905         void *obj_handle;
1906
1907         CHECK(p, EINVAL);
1908
1909         CHECK_NAME(extern_type_name, EINVAL);
1910         type = extern_type_find(p, extern_type_name);
1911         CHECK(type, EINVAL);
1912
1913         CHECK_NAME(name, EINVAL);
1914         CHECK(!extern_obj_find(p, name), EEXIST);
1915
1916         /* Node allocation. */
1917         obj = calloc(1, sizeof(struct extern_obj));
1918         CHECK(obj, ENOMEM);
1919
1920         /* Object construction. */
1921         obj_handle = type->constructor(args);
1922         if (!obj_handle) {
1923                 free(obj);
1924                 CHECK(0, ENODEV);
1925         }
1926
1927         /* Node initialization. */
1928         strcpy(obj->name, name);
1929         obj->type = type;
1930         obj->obj = obj_handle;
1931         obj->struct_id = p->n_structs;
1932         obj->id = p->n_extern_objs;
1933
1934         /* Node add to tailq. */
1935         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1936         p->n_extern_objs++;
1937         p->n_structs++;
1938
1939         return 0;
1940 }
1941
1942 static int
1943 extern_obj_build(struct rte_swx_pipeline *p)
1944 {
1945         uint32_t i;
1946
1947         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1948                 struct thread *t = &p->threads[i];
1949                 struct extern_obj *obj;
1950
1951                 t->extern_objs = calloc(p->n_extern_objs,
1952                                         sizeof(struct extern_obj_runtime));
1953                 CHECK(t->extern_objs, ENOMEM);
1954
1955                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1956                         struct extern_obj_runtime *r =
1957                                 &t->extern_objs[obj->id];
1958                         struct extern_type_member_func *func;
1959                         uint32_t mailbox_size =
1960                                 obj->type->mailbox_struct_type->n_bits / 8;
1961
1962                         r->obj = obj->obj;
1963
1964                         r->mailbox = calloc(1, mailbox_size);
1965                         CHECK(r->mailbox, ENOMEM);
1966
1967                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1968                                 r->funcs[func->id] = func->func;
1969
1970                         t->structs[obj->struct_id] = r->mailbox;
1971                 }
1972         }
1973
1974         return 0;
1975 }
1976
1977 static void
1978 extern_obj_build_free(struct rte_swx_pipeline *p)
1979 {
1980         uint32_t i;
1981
1982         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1983                 struct thread *t = &p->threads[i];
1984                 uint32_t j;
1985
1986                 if (!t->extern_objs)
1987                         continue;
1988
1989                 for (j = 0; j < p->n_extern_objs; j++) {
1990                         struct extern_obj_runtime *r = &t->extern_objs[j];
1991
1992                         free(r->mailbox);
1993                 }
1994
1995                 free(t->extern_objs);
1996                 t->extern_objs = NULL;
1997         }
1998 }
1999
2000 static void
2001 extern_obj_free(struct rte_swx_pipeline *p)
2002 {
2003         extern_obj_build_free(p);
2004
2005         /* Extern objects. */
2006         for ( ; ; ) {
2007                 struct extern_obj *elem;
2008
2009                 elem = TAILQ_FIRST(&p->extern_objs);
2010                 if (!elem)
2011                         break;
2012
2013                 TAILQ_REMOVE(&p->extern_objs, elem, node);
2014                 if (elem->obj)
2015                         elem->type->destructor(elem->obj);
2016                 free(elem);
2017         }
2018
2019         /* Extern types. */
2020         for ( ; ; ) {
2021                 struct extern_type *elem;
2022
2023                 elem = TAILQ_FIRST(&p->extern_types);
2024                 if (!elem)
2025                         break;
2026
2027                 TAILQ_REMOVE(&p->extern_types, elem, node);
2028
2029                 for ( ; ; ) {
2030                         struct extern_type_member_func *func;
2031
2032                         func = TAILQ_FIRST(&elem->funcs);
2033                         if (!func)
2034                                 break;
2035
2036                         TAILQ_REMOVE(&elem->funcs, func, node);
2037                         free(func);
2038                 }
2039
2040                 free(elem);
2041         }
2042 }
2043
2044 /*
2045  * Extern function.
2046  */
2047 static struct extern_func *
2048 extern_func_find(struct rte_swx_pipeline *p, const char *name)
2049 {
2050         struct extern_func *elem;
2051
2052         TAILQ_FOREACH(elem, &p->extern_funcs, node)
2053                 if (strcmp(elem->name, name) == 0)
2054                         return elem;
2055
2056         return NULL;
2057 }
2058
2059 static struct extern_func *
2060 extern_func_parse(struct rte_swx_pipeline *p,
2061                   const char *name)
2062 {
2063         if (name[0] != 'f' || name[1] != '.')
2064                 return NULL;
2065
2066         return extern_func_find(p, &name[2]);
2067 }
2068
2069 static struct field *
2070 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
2071                                 const char *name,
2072                                 struct extern_func **function)
2073 {
2074         struct extern_func *func;
2075         struct field *f;
2076         char *func_name, *field_name;
2077
2078         if ((name[0] != 'f') || (name[1] != '.'))
2079                 return NULL;
2080
2081         func_name = strdup(&name[2]);
2082         if (!func_name)
2083                 return NULL;
2084
2085         field_name = strchr(func_name, '.');
2086         if (!field_name) {
2087                 free(func_name);
2088                 return NULL;
2089         }
2090
2091         *field_name = 0;
2092         field_name++;
2093
2094         func = extern_func_find(p, func_name);
2095         if (!func) {
2096                 free(func_name);
2097                 return NULL;
2098         }
2099
2100         f = struct_type_field_find(func->mailbox_struct_type, field_name);
2101         if (!f) {
2102                 free(func_name);
2103                 return NULL;
2104         }
2105
2106         if (function)
2107                 *function = func;
2108
2109         free(func_name);
2110         return f;
2111 }
2112
2113 int
2114 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
2115                                       const char *name,
2116                                       const char *mailbox_struct_type_name,
2117                                       rte_swx_extern_func_t func)
2118 {
2119         struct extern_func *f;
2120         struct struct_type *mailbox_struct_type;
2121
2122         CHECK(p, EINVAL);
2123
2124         CHECK_NAME(name, EINVAL);
2125         CHECK(!extern_func_find(p, name), EEXIST);
2126
2127         CHECK_NAME(mailbox_struct_type_name, EINVAL);
2128         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2129         CHECK(mailbox_struct_type, EINVAL);
2130
2131         CHECK(func, EINVAL);
2132
2133         /* Node allocation. */
2134         f = calloc(1, sizeof(struct extern_func));
2135         CHECK(func, ENOMEM);
2136
2137         /* Node initialization. */
2138         strcpy(f->name, name);
2139         f->mailbox_struct_type = mailbox_struct_type;
2140         f->func = func;
2141         f->struct_id = p->n_structs;
2142         f->id = p->n_extern_funcs;
2143
2144         /* Node add to tailq. */
2145         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
2146         p->n_extern_funcs++;
2147         p->n_structs++;
2148
2149         return 0;
2150 }
2151
2152 static int
2153 extern_func_build(struct rte_swx_pipeline *p)
2154 {
2155         uint32_t i;
2156
2157         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2158                 struct thread *t = &p->threads[i];
2159                 struct extern_func *func;
2160
2161                 /* Memory allocation. */
2162                 t->extern_funcs = calloc(p->n_extern_funcs,
2163                                          sizeof(struct extern_func_runtime));
2164                 CHECK(t->extern_funcs, ENOMEM);
2165
2166                 /* Extern function. */
2167                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
2168                         struct extern_func_runtime *r =
2169                                 &t->extern_funcs[func->id];
2170                         uint32_t mailbox_size =
2171                                 func->mailbox_struct_type->n_bits / 8;
2172
2173                         r->func = func->func;
2174
2175                         r->mailbox = calloc(1, mailbox_size);
2176                         CHECK(r->mailbox, ENOMEM);
2177
2178                         t->structs[func->struct_id] = r->mailbox;
2179                 }
2180         }
2181
2182         return 0;
2183 }
2184
2185 static void
2186 extern_func_build_free(struct rte_swx_pipeline *p)
2187 {
2188         uint32_t i;
2189
2190         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2191                 struct thread *t = &p->threads[i];
2192                 uint32_t j;
2193
2194                 if (!t->extern_funcs)
2195                         continue;
2196
2197                 for (j = 0; j < p->n_extern_funcs; j++) {
2198                         struct extern_func_runtime *r = &t->extern_funcs[j];
2199
2200                         free(r->mailbox);
2201                 }
2202
2203                 free(t->extern_funcs);
2204                 t->extern_funcs = NULL;
2205         }
2206 }
2207
2208 static void
2209 extern_func_free(struct rte_swx_pipeline *p)
2210 {
2211         extern_func_build_free(p);
2212
2213         for ( ; ; ) {
2214                 struct extern_func *elem;
2215
2216                 elem = TAILQ_FIRST(&p->extern_funcs);
2217                 if (!elem)
2218                         break;
2219
2220                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
2221                 free(elem);
2222         }
2223 }
2224
2225 /*
2226  * Header.
2227  */
2228 static struct header *
2229 header_find(struct rte_swx_pipeline *p, const char *name)
2230 {
2231         struct header *elem;
2232
2233         TAILQ_FOREACH(elem, &p->headers, node)
2234                 if (strcmp(elem->name, name) == 0)
2235                         return elem;
2236
2237         return NULL;
2238 }
2239
2240 static struct header *
2241 header_parse(struct rte_swx_pipeline *p,
2242              const char *name)
2243 {
2244         if (name[0] != 'h' || name[1] != '.')
2245                 return NULL;
2246
2247         return header_find(p, &name[2]);
2248 }
2249
2250 static struct field *
2251 header_field_parse(struct rte_swx_pipeline *p,
2252                    const char *name,
2253                    struct header **header)
2254 {
2255         struct header *h;
2256         struct field *f;
2257         char *header_name, *field_name;
2258
2259         if ((name[0] != 'h') || (name[1] != '.'))
2260                 return NULL;
2261
2262         header_name = strdup(&name[2]);
2263         if (!header_name)
2264                 return NULL;
2265
2266         field_name = strchr(header_name, '.');
2267         if (!field_name) {
2268                 free(header_name);
2269                 return NULL;
2270         }
2271
2272         *field_name = 0;
2273         field_name++;
2274
2275         h = header_find(p, header_name);
2276         if (!h) {
2277                 free(header_name);
2278                 return NULL;
2279         }
2280
2281         f = struct_type_field_find(h->st, field_name);
2282         if (!f) {
2283                 free(header_name);
2284                 return NULL;
2285         }
2286
2287         if (header)
2288                 *header = h;
2289
2290         free(header_name);
2291         return f;
2292 }
2293
2294 int
2295 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2296                                         const char *name,
2297                                         const char *struct_type_name)
2298 {
2299         struct struct_type *st;
2300         struct header *h;
2301         size_t n_headers_max;
2302
2303         CHECK(p, EINVAL);
2304         CHECK_NAME(name, EINVAL);
2305         CHECK_NAME(struct_type_name, EINVAL);
2306
2307         CHECK(!header_find(p, name), EEXIST);
2308
2309         st = struct_type_find(p, struct_type_name);
2310         CHECK(st, EINVAL);
2311
2312         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2313         CHECK(p->n_headers < n_headers_max, ENOSPC);
2314
2315         /* Node allocation. */
2316         h = calloc(1, sizeof(struct header));
2317         CHECK(h, ENOMEM);
2318
2319         /* Node initialization. */
2320         strcpy(h->name, name);
2321         h->st = st;
2322         h->struct_id = p->n_structs;
2323         h->id = p->n_headers;
2324
2325         /* Node add to tailq. */
2326         TAILQ_INSERT_TAIL(&p->headers, h, node);
2327         p->n_headers++;
2328         p->n_structs++;
2329
2330         return 0;
2331 }
2332
2333 static int
2334 header_build(struct rte_swx_pipeline *p)
2335 {
2336         struct header *h;
2337         uint32_t n_bytes = 0, i;
2338
2339         TAILQ_FOREACH(h, &p->headers, node) {
2340                 n_bytes += h->st->n_bits / 8;
2341         }
2342
2343         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2344                 struct thread *t = &p->threads[i];
2345                 uint32_t offset = 0;
2346
2347                 t->headers = calloc(p->n_headers,
2348                                     sizeof(struct header_runtime));
2349                 CHECK(t->headers, ENOMEM);
2350
2351                 t->headers_out = calloc(p->n_headers,
2352                                         sizeof(struct header_out_runtime));
2353                 CHECK(t->headers_out, ENOMEM);
2354
2355                 t->header_storage = calloc(1, n_bytes);
2356                 CHECK(t->header_storage, ENOMEM);
2357
2358                 t->header_out_storage = calloc(1, n_bytes);
2359                 CHECK(t->header_out_storage, ENOMEM);
2360
2361                 TAILQ_FOREACH(h, &p->headers, node) {
2362                         uint8_t *header_storage;
2363
2364                         header_storage = &t->header_storage[offset];
2365                         offset += h->st->n_bits / 8;
2366
2367                         t->headers[h->id].ptr0 = header_storage;
2368                         t->structs[h->struct_id] = header_storage;
2369                 }
2370         }
2371
2372         return 0;
2373 }
2374
2375 static void
2376 header_build_free(struct rte_swx_pipeline *p)
2377 {
2378         uint32_t i;
2379
2380         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2381                 struct thread *t = &p->threads[i];
2382
2383                 free(t->headers_out);
2384                 t->headers_out = NULL;
2385
2386                 free(t->headers);
2387                 t->headers = NULL;
2388
2389                 free(t->header_out_storage);
2390                 t->header_out_storage = NULL;
2391
2392                 free(t->header_storage);
2393                 t->header_storage = NULL;
2394         }
2395 }
2396
2397 static void
2398 header_free(struct rte_swx_pipeline *p)
2399 {
2400         header_build_free(p);
2401
2402         for ( ; ; ) {
2403                 struct header *elem;
2404
2405                 elem = TAILQ_FIRST(&p->headers);
2406                 if (!elem)
2407                         break;
2408
2409                 TAILQ_REMOVE(&p->headers, elem, node);
2410                 free(elem);
2411         }
2412 }
2413
2414 /*
2415  * Meta-data.
2416  */
2417 static struct field *
2418 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2419 {
2420         if (!p->metadata_st)
2421                 return NULL;
2422
2423         if (name[0] != 'm' || name[1] != '.')
2424                 return NULL;
2425
2426         return struct_type_field_find(p->metadata_st, &name[2]);
2427 }
2428
2429 int
2430 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2431                                           const char *struct_type_name)
2432 {
2433         struct struct_type *st = NULL;
2434
2435         CHECK(p, EINVAL);
2436
2437         CHECK_NAME(struct_type_name, EINVAL);
2438         st  = struct_type_find(p, struct_type_name);
2439         CHECK(st, EINVAL);
2440         CHECK(!p->metadata_st, EINVAL);
2441
2442         p->metadata_st = st;
2443         p->metadata_struct_id = p->n_structs;
2444
2445         p->n_structs++;
2446
2447         return 0;
2448 }
2449
2450 static int
2451 metadata_build(struct rte_swx_pipeline *p)
2452 {
2453         uint32_t n_bytes = p->metadata_st->n_bits / 8;
2454         uint32_t i;
2455
2456         /* Thread-level initialization. */
2457         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2458                 struct thread *t = &p->threads[i];
2459                 uint8_t *metadata;
2460
2461                 metadata = calloc(1, n_bytes);
2462                 CHECK(metadata, ENOMEM);
2463
2464                 t->metadata = metadata;
2465                 t->structs[p->metadata_struct_id] = metadata;
2466         }
2467
2468         return 0;
2469 }
2470
2471 static void
2472 metadata_build_free(struct rte_swx_pipeline *p)
2473 {
2474         uint32_t i;
2475
2476         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2477                 struct thread *t = &p->threads[i];
2478
2479                 free(t->metadata);
2480                 t->metadata = NULL;
2481         }
2482 }
2483
2484 static void
2485 metadata_free(struct rte_swx_pipeline *p)
2486 {
2487         metadata_build_free(p);
2488 }
2489
2490 /*
2491  * Instruction.
2492  */
2493 static int
2494 instruction_is_jmp(struct instruction *instr)
2495 {
2496         switch (instr->type) {
2497         case INSTR_JMP:
2498         case INSTR_JMP_VALID:
2499         case INSTR_JMP_INVALID:
2500         case INSTR_JMP_HIT:
2501         case INSTR_JMP_MISS:
2502         case INSTR_JMP_ACTION_HIT:
2503         case INSTR_JMP_ACTION_MISS:
2504         case INSTR_JMP_EQ:
2505         case INSTR_JMP_EQ_S:
2506         case INSTR_JMP_EQ_I:
2507         case INSTR_JMP_NEQ:
2508         case INSTR_JMP_NEQ_S:
2509         case INSTR_JMP_NEQ_I:
2510         case INSTR_JMP_LT:
2511         case INSTR_JMP_LT_MH:
2512         case INSTR_JMP_LT_HM:
2513         case INSTR_JMP_LT_HH:
2514         case INSTR_JMP_LT_MI:
2515         case INSTR_JMP_LT_HI:
2516         case INSTR_JMP_GT:
2517         case INSTR_JMP_GT_MH:
2518         case INSTR_JMP_GT_HM:
2519         case INSTR_JMP_GT_HH:
2520         case INSTR_JMP_GT_MI:
2521         case INSTR_JMP_GT_HI:
2522                 return 1;
2523
2524         default:
2525                 return 0;
2526         }
2527 }
2528
2529 static struct field *
2530 action_field_parse(struct action *action, const char *name);
2531
2532 static struct field *
2533 struct_field_parse(struct rte_swx_pipeline *p,
2534                    struct action *action,
2535                    const char *name,
2536                    uint32_t *struct_id)
2537 {
2538         struct field *f;
2539
2540         switch (name[0]) {
2541         case 'h':
2542         {
2543                 struct header *header;
2544
2545                 f = header_field_parse(p, name, &header);
2546                 if (!f)
2547                         return NULL;
2548
2549                 *struct_id = header->struct_id;
2550                 return f;
2551         }
2552
2553         case 'm':
2554         {
2555                 f = metadata_field_parse(p, name);
2556                 if (!f)
2557                         return NULL;
2558
2559                 *struct_id = p->metadata_struct_id;
2560                 return f;
2561         }
2562
2563         case 't':
2564         {
2565                 if (!action)
2566                         return NULL;
2567
2568                 f = action_field_parse(action, name);
2569                 if (!f)
2570                         return NULL;
2571
2572                 *struct_id = 0;
2573                 return f;
2574         }
2575
2576         case 'e':
2577         {
2578                 struct extern_obj *obj;
2579
2580                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2581                 if (!f)
2582                         return NULL;
2583
2584                 *struct_id = obj->struct_id;
2585                 return f;
2586         }
2587
2588         case 'f':
2589         {
2590                 struct extern_func *func;
2591
2592                 f = extern_func_mailbox_field_parse(p, name, &func);
2593                 if (!f)
2594                         return NULL;
2595
2596                 *struct_id = func->struct_id;
2597                 return f;
2598         }
2599
2600         default:
2601                 return NULL;
2602         }
2603 }
2604
2605 static inline void
2606 pipeline_port_inc(struct rte_swx_pipeline *p)
2607 {
2608         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2609 }
2610
2611 static inline void
2612 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2613 {
2614         t->ip = p->instructions;
2615 }
2616
2617 static inline void
2618 thread_ip_set(struct thread *t, struct instruction *ip)
2619 {
2620         t->ip = ip;
2621 }
2622
2623 static inline void
2624 thread_ip_action_call(struct rte_swx_pipeline *p,
2625                       struct thread *t,
2626                       uint32_t action_id)
2627 {
2628         t->ret = t->ip + 1;
2629         t->ip = p->action_instructions[action_id];
2630 }
2631
2632 static inline void
2633 thread_ip_inc(struct rte_swx_pipeline *p);
2634
2635 static inline void
2636 thread_ip_inc(struct rte_swx_pipeline *p)
2637 {
2638         struct thread *t = &p->threads[p->thread_id];
2639
2640         t->ip++;
2641 }
2642
2643 static inline void
2644 thread_ip_inc_cond(struct thread *t, int cond)
2645 {
2646         t->ip += cond;
2647 }
2648
2649 static inline void
2650 thread_yield(struct rte_swx_pipeline *p)
2651 {
2652         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2653 }
2654
2655 static inline void
2656 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2657 {
2658         p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2659 }
2660
2661 /*
2662  * rx.
2663  */
2664 static int
2665 instr_rx_translate(struct rte_swx_pipeline *p,
2666                    struct action *action,
2667                    char **tokens,
2668                    int n_tokens,
2669                    struct instruction *instr,
2670                    struct instruction_data *data __rte_unused)
2671 {
2672         struct field *f;
2673
2674         CHECK(!action, EINVAL);
2675         CHECK(n_tokens == 2, EINVAL);
2676
2677         f = metadata_field_parse(p, tokens[1]);
2678         CHECK(f, EINVAL);
2679
2680         instr->type = INSTR_RX;
2681         instr->io.io.offset = f->offset / 8;
2682         instr->io.io.n_bits = f->n_bits;
2683         return 0;
2684 }
2685
2686 static inline void
2687 instr_rx_exec(struct rte_swx_pipeline *p);
2688
2689 static inline void
2690 instr_rx_exec(struct rte_swx_pipeline *p)
2691 {
2692         struct thread *t = &p->threads[p->thread_id];
2693         struct instruction *ip = t->ip;
2694         struct port_in_runtime *port = &p->in[p->port_id];
2695         struct rte_swx_pkt *pkt = &t->pkt;
2696         int pkt_received;
2697
2698         /* Packet. */
2699         pkt_received = port->pkt_rx(port->obj, pkt);
2700         t->ptr = &pkt->pkt[pkt->offset];
2701         rte_prefetch0(t->ptr);
2702
2703         TRACE("[Thread %2u] rx %s from port %u\n",
2704               p->thread_id,
2705               pkt_received ? "1 pkt" : "0 pkts",
2706               p->port_id);
2707
2708         /* Headers. */
2709         t->valid_headers = 0;
2710         t->n_headers_out = 0;
2711
2712         /* Meta-data. */
2713         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2714
2715         /* Tables. */
2716         t->table_state = p->table_state;
2717
2718         /* Thread. */
2719         pipeline_port_inc(p);
2720         thread_ip_inc_cond(t, pkt_received);
2721         thread_yield(p);
2722 }
2723
2724 /*
2725  * tx.
2726  */
2727 static int
2728 instr_tx_translate(struct rte_swx_pipeline *p,
2729                    struct action *action __rte_unused,
2730                    char **tokens,
2731                    int n_tokens,
2732                    struct instruction *instr,
2733                    struct instruction_data *data __rte_unused)
2734 {
2735         struct field *f;
2736
2737         CHECK(n_tokens == 2, EINVAL);
2738
2739         f = metadata_field_parse(p, tokens[1]);
2740         CHECK(f, EINVAL);
2741
2742         instr->type = INSTR_TX;
2743         instr->io.io.offset = f->offset / 8;
2744         instr->io.io.n_bits = f->n_bits;
2745         return 0;
2746 }
2747
2748 static inline void
2749 emit_handler(struct thread *t)
2750 {
2751         struct header_out_runtime *h0 = &t->headers_out[0];
2752         struct header_out_runtime *h1 = &t->headers_out[1];
2753         uint32_t offset = 0, i;
2754
2755         /* No header change or header decapsulation. */
2756         if ((t->n_headers_out == 1) &&
2757             (h0->ptr + h0->n_bytes == t->ptr)) {
2758                 TRACE("Emit handler: no header change or header decap.\n");
2759
2760                 t->pkt.offset -= h0->n_bytes;
2761                 t->pkt.length += h0->n_bytes;
2762
2763                 return;
2764         }
2765
2766         /* Header encapsulation (optionally, with prior header decasulation). */
2767         if ((t->n_headers_out == 2) &&
2768             (h1->ptr + h1->n_bytes == t->ptr) &&
2769             (h0->ptr == h0->ptr0)) {
2770                 uint32_t offset;
2771
2772                 TRACE("Emit handler: header encapsulation.\n");
2773
2774                 offset = h0->n_bytes + h1->n_bytes;
2775                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2776                 t->pkt.offset -= offset;
2777                 t->pkt.length += offset;
2778
2779                 return;
2780         }
2781
2782         /* Header insertion. */
2783         /* TBD */
2784
2785         /* Header extraction. */
2786         /* TBD */
2787
2788         /* For any other case. */
2789         TRACE("Emit handler: complex case.\n");
2790
2791         for (i = 0; i < t->n_headers_out; i++) {
2792                 struct header_out_runtime *h = &t->headers_out[i];
2793
2794                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2795                 offset += h->n_bytes;
2796         }
2797
2798         if (offset) {
2799                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2800                 t->pkt.offset -= offset;
2801                 t->pkt.length += offset;
2802         }
2803 }
2804
2805 static inline void
2806 instr_tx_exec(struct rte_swx_pipeline *p);
2807
2808 static inline void
2809 instr_tx_exec(struct rte_swx_pipeline *p)
2810 {
2811         struct thread *t = &p->threads[p->thread_id];
2812         struct instruction *ip = t->ip;
2813         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2814         struct port_out_runtime *port = &p->out[port_id];
2815         struct rte_swx_pkt *pkt = &t->pkt;
2816
2817         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2818               p->thread_id,
2819               (uint32_t)port_id);
2820
2821         /* Headers. */
2822         emit_handler(t);
2823
2824         /* Packet. */
2825         port->pkt_tx(port->obj, pkt);
2826
2827         /* Thread. */
2828         thread_ip_reset(p, t);
2829         instr_rx_exec(p);
2830 }
2831
2832 /*
2833  * extract.
2834  */
2835 static int
2836 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2837                             struct action *action,
2838                             char **tokens,
2839                             int n_tokens,
2840                             struct instruction *instr,
2841                             struct instruction_data *data __rte_unused)
2842 {
2843         struct header *h;
2844
2845         CHECK(!action, EINVAL);
2846         CHECK(n_tokens == 2, EINVAL);
2847
2848         h = header_parse(p, tokens[1]);
2849         CHECK(h, EINVAL);
2850
2851         instr->type = INSTR_HDR_EXTRACT;
2852         instr->io.hdr.header_id[0] = h->id;
2853         instr->io.hdr.struct_id[0] = h->struct_id;
2854         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2855         return 0;
2856 }
2857
2858 static inline void
2859 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2860
2861 static inline void
2862 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2863 {
2864         struct thread *t = &p->threads[p->thread_id];
2865         struct instruction *ip = t->ip;
2866         uint64_t valid_headers = t->valid_headers;
2867         uint8_t *ptr = t->ptr;
2868         uint32_t offset = t->pkt.offset;
2869         uint32_t length = t->pkt.length;
2870         uint32_t i;
2871
2872         for (i = 0; i < n_extract; i++) {
2873                 uint32_t header_id = ip->io.hdr.header_id[i];
2874                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2875                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2876
2877                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2878                       p->thread_id,
2879                       header_id,
2880                       n_bytes);
2881
2882                 /* Headers. */
2883                 t->structs[struct_id] = ptr;
2884                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2885
2886                 /* Packet. */
2887                 offset += n_bytes;
2888                 length -= n_bytes;
2889                 ptr += n_bytes;
2890         }
2891
2892         /* Headers. */
2893         t->valid_headers = valid_headers;
2894
2895         /* Packet. */
2896         t->pkt.offset = offset;
2897         t->pkt.length = length;
2898         t->ptr = ptr;
2899 }
2900
2901 static inline void
2902 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2903 {
2904         __instr_hdr_extract_exec(p, 1);
2905
2906         /* Thread. */
2907         thread_ip_inc(p);
2908 }
2909
2910 static inline void
2911 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2912 {
2913         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2914               p->thread_id);
2915
2916         __instr_hdr_extract_exec(p, 2);
2917
2918         /* Thread. */
2919         thread_ip_inc(p);
2920 }
2921
2922 static inline void
2923 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2924 {
2925         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2926               p->thread_id);
2927
2928         __instr_hdr_extract_exec(p, 3);
2929
2930         /* Thread. */
2931         thread_ip_inc(p);
2932 }
2933
2934 static inline void
2935 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2936 {
2937         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2938               p->thread_id);
2939
2940         __instr_hdr_extract_exec(p, 4);
2941
2942         /* Thread. */
2943         thread_ip_inc(p);
2944 }
2945
2946 static inline void
2947 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2948 {
2949         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2950               p->thread_id);
2951
2952         __instr_hdr_extract_exec(p, 5);
2953
2954         /* Thread. */
2955         thread_ip_inc(p);
2956 }
2957
2958 static inline void
2959 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2960 {
2961         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2962               p->thread_id);
2963
2964         __instr_hdr_extract_exec(p, 6);
2965
2966         /* Thread. */
2967         thread_ip_inc(p);
2968 }
2969
2970 static inline void
2971 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2972 {
2973         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2974               p->thread_id);
2975
2976         __instr_hdr_extract_exec(p, 7);
2977
2978         /* Thread. */
2979         thread_ip_inc(p);
2980 }
2981
2982 static inline void
2983 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2984 {
2985         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2986               p->thread_id);
2987
2988         __instr_hdr_extract_exec(p, 8);
2989
2990         /* Thread. */
2991         thread_ip_inc(p);
2992 }
2993
2994 /*
2995  * emit.
2996  */
2997 static int
2998 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2999                          struct action *action __rte_unused,
3000                          char **tokens,
3001                          int n_tokens,
3002                          struct instruction *instr,
3003                          struct instruction_data *data __rte_unused)
3004 {
3005         struct header *h;
3006
3007         CHECK(n_tokens == 2, EINVAL);
3008
3009         h = header_parse(p, tokens[1]);
3010         CHECK(h, EINVAL);
3011
3012         instr->type = INSTR_HDR_EMIT;
3013         instr->io.hdr.header_id[0] = h->id;
3014         instr->io.hdr.struct_id[0] = h->struct_id;
3015         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3016         return 0;
3017 }
3018
3019 static inline void
3020 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
3021
3022 static inline void
3023 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
3024 {
3025         struct thread *t = &p->threads[p->thread_id];
3026         struct instruction *ip = t->ip;
3027         uint32_t n_headers_out = t->n_headers_out;
3028         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
3029         uint8_t *ho_ptr = NULL;
3030         uint32_t ho_nbytes = 0, i;
3031
3032         for (i = 0; i < n_emit; i++) {
3033                 uint32_t header_id = ip->io.hdr.header_id[i];
3034                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3035                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3036
3037                 struct header_runtime *hi = &t->headers[header_id];
3038                 uint8_t *hi_ptr = t->structs[struct_id];
3039
3040                 TRACE("[Thread %2u]: emit header %u\n",
3041                       p->thread_id,
3042                       header_id);
3043
3044                 /* Headers. */
3045                 if (!i) {
3046                         if (!t->n_headers_out) {
3047                                 ho = &t->headers_out[0];
3048
3049                                 ho->ptr0 = hi->ptr0;
3050                                 ho->ptr = hi_ptr;
3051
3052                                 ho_ptr = hi_ptr;
3053                                 ho_nbytes = n_bytes;
3054
3055                                 n_headers_out = 1;
3056
3057                                 continue;
3058                         } else {
3059                                 ho_ptr = ho->ptr;
3060                                 ho_nbytes = ho->n_bytes;
3061                         }
3062                 }
3063
3064                 if (ho_ptr + ho_nbytes == hi_ptr) {
3065                         ho_nbytes += n_bytes;
3066                 } else {
3067                         ho->n_bytes = ho_nbytes;
3068
3069                         ho++;
3070                         ho->ptr0 = hi->ptr0;
3071                         ho->ptr = hi_ptr;
3072
3073                         ho_ptr = hi_ptr;
3074                         ho_nbytes = n_bytes;
3075
3076                         n_headers_out++;
3077                 }
3078         }
3079
3080         ho->n_bytes = ho_nbytes;
3081         t->n_headers_out = n_headers_out;
3082 }
3083
3084 static inline void
3085 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
3086 {
3087         __instr_hdr_emit_exec(p, 1);
3088
3089         /* Thread. */
3090         thread_ip_inc(p);
3091 }
3092
3093 static inline void
3094 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
3095 {
3096         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3097               p->thread_id);
3098
3099         __instr_hdr_emit_exec(p, 1);
3100         instr_tx_exec(p);
3101 }
3102
3103 static inline void
3104 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
3105 {
3106         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3107               p->thread_id);
3108
3109         __instr_hdr_emit_exec(p, 2);
3110         instr_tx_exec(p);
3111 }
3112
3113 static inline void
3114 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
3115 {
3116         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3117               p->thread_id);
3118
3119         __instr_hdr_emit_exec(p, 3);
3120         instr_tx_exec(p);
3121 }
3122
3123 static inline void
3124 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
3125 {
3126         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3127               p->thread_id);
3128
3129         __instr_hdr_emit_exec(p, 4);
3130         instr_tx_exec(p);
3131 }
3132
3133 static inline void
3134 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
3135 {
3136         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3137               p->thread_id);
3138
3139         __instr_hdr_emit_exec(p, 5);
3140         instr_tx_exec(p);
3141 }
3142
3143 static inline void
3144 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
3145 {
3146         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3147               p->thread_id);
3148
3149         __instr_hdr_emit_exec(p, 6);
3150         instr_tx_exec(p);
3151 }
3152
3153 static inline void
3154 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
3155 {
3156         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3157               p->thread_id);
3158
3159         __instr_hdr_emit_exec(p, 7);
3160         instr_tx_exec(p);
3161 }
3162
3163 static inline void
3164 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
3165 {
3166         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
3167               p->thread_id);
3168
3169         __instr_hdr_emit_exec(p, 8);
3170         instr_tx_exec(p);
3171 }
3172
3173 /*
3174  * validate.
3175  */
3176 static int
3177 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
3178                              struct action *action __rte_unused,
3179                              char **tokens,
3180                              int n_tokens,
3181                              struct instruction *instr,
3182                              struct instruction_data *data __rte_unused)
3183 {
3184         struct header *h;
3185
3186         CHECK(n_tokens == 2, EINVAL);
3187
3188         h = header_parse(p, tokens[1]);
3189         CHECK(h, EINVAL);
3190
3191         instr->type = INSTR_HDR_VALIDATE;
3192         instr->valid.header_id = h->id;
3193         return 0;
3194 }
3195
3196 static inline void
3197 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
3198 {
3199         struct thread *t = &p->threads[p->thread_id];
3200         struct instruction *ip = t->ip;
3201         uint32_t header_id = ip->valid.header_id;
3202
3203         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
3204
3205         /* Headers. */
3206         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
3207
3208         /* Thread. */
3209         thread_ip_inc(p);
3210 }
3211
3212 /*
3213  * invalidate.
3214  */
3215 static int
3216 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
3217                                struct action *action __rte_unused,
3218                                char **tokens,
3219                                int n_tokens,
3220                                struct instruction *instr,
3221                                struct instruction_data *data __rte_unused)
3222 {
3223         struct header *h;
3224
3225         CHECK(n_tokens == 2, EINVAL);
3226
3227         h = header_parse(p, tokens[1]);
3228         CHECK(h, EINVAL);
3229
3230         instr->type = INSTR_HDR_INVALIDATE;
3231         instr->valid.header_id = h->id;
3232         return 0;
3233 }
3234
3235 static inline void
3236 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3237 {
3238         struct thread *t = &p->threads[p->thread_id];
3239         struct instruction *ip = t->ip;
3240         uint32_t header_id = ip->valid.header_id;
3241
3242         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3243
3244         /* Headers. */
3245         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3246
3247         /* Thread. */
3248         thread_ip_inc(p);
3249 }
3250
3251 /*
3252  * table.
3253  */
3254 static struct table *
3255 table_find(struct rte_swx_pipeline *p, const char *name);
3256
3257 static int
3258 instr_table_translate(struct rte_swx_pipeline *p,
3259                       struct action *action,
3260                       char **tokens,
3261                       int n_tokens,
3262                       struct instruction *instr,
3263                       struct instruction_data *data __rte_unused)
3264 {
3265         struct table *t;
3266
3267         CHECK(!action, EINVAL);
3268         CHECK(n_tokens == 2, EINVAL);
3269
3270         t = table_find(p, tokens[1]);
3271         CHECK(t, EINVAL);
3272
3273         instr->type = INSTR_TABLE;
3274         instr->table.table_id = t->id;
3275         return 0;
3276 }
3277
3278 static inline void
3279 instr_table_exec(struct rte_swx_pipeline *p)
3280 {
3281         struct thread *t = &p->threads[p->thread_id];
3282         struct instruction *ip = t->ip;
3283         uint32_t table_id = ip->table.table_id;
3284         struct rte_swx_table_state *ts = &t->table_state[table_id];
3285         struct table_runtime *table = &t->tables[table_id];
3286         uint64_t action_id;
3287         uint8_t *action_data;
3288         int done, hit;
3289
3290         /* Table. */
3291         done = table->func(ts->obj,
3292                            table->mailbox,
3293                            table->key,
3294                            &action_id,
3295                            &action_data,
3296                            &hit);
3297         if (!done) {
3298                 /* Thread. */
3299                 TRACE("[Thread %2u] table %u (not finalized)\n",
3300                       p->thread_id,
3301                       table_id);
3302
3303                 thread_yield(p);
3304                 return;
3305         }
3306
3307         action_id = hit ? action_id : ts->default_action_id;
3308         action_data = hit ? action_data : ts->default_action_data;
3309
3310         TRACE("[Thread %2u] table %u (%s, action %u)\n",
3311               p->thread_id,
3312               table_id,
3313               hit ? "hit" : "miss",
3314               (uint32_t)action_id);
3315
3316         t->action_id = action_id;
3317         t->structs[0] = action_data;
3318         t->hit = hit;
3319
3320         /* Thread. */
3321         thread_ip_action_call(p, t, action_id);
3322 }
3323
3324 /*
3325  * extern.
3326  */
3327 static int
3328 instr_extern_translate(struct rte_swx_pipeline *p,
3329                        struct action *action __rte_unused,
3330                        char **tokens,
3331                        int n_tokens,
3332                        struct instruction *instr,
3333                        struct instruction_data *data __rte_unused)
3334 {
3335         char *token = tokens[1];
3336
3337         CHECK(n_tokens == 2, EINVAL);
3338
3339         if (token[0] == 'e') {
3340                 struct extern_obj *obj;
3341                 struct extern_type_member_func *func;
3342
3343                 func = extern_obj_member_func_parse(p, token, &obj);
3344                 CHECK(func, EINVAL);
3345
3346                 instr->type = INSTR_EXTERN_OBJ;
3347                 instr->ext_obj.ext_obj_id = obj->id;
3348                 instr->ext_obj.func_id = func->id;
3349
3350                 return 0;
3351         }
3352
3353         if (token[0] == 'f') {
3354                 struct extern_func *func;
3355
3356                 func = extern_func_parse(p, token);
3357                 CHECK(func, EINVAL);
3358
3359                 instr->type = INSTR_EXTERN_FUNC;
3360                 instr->ext_func.ext_func_id = func->id;
3361
3362                 return 0;
3363         }
3364
3365         CHECK(0, EINVAL);
3366 }
3367
3368 static inline void
3369 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3370 {
3371         struct thread *t = &p->threads[p->thread_id];
3372         struct instruction *ip = t->ip;
3373         uint32_t obj_id = ip->ext_obj.ext_obj_id;
3374         uint32_t func_id = ip->ext_obj.func_id;
3375         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3376         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3377
3378         TRACE("[Thread %2u] extern obj %u member func %u\n",
3379               p->thread_id,
3380               obj_id,
3381               func_id);
3382
3383         /* Extern object member function execute. */
3384         uint32_t done = func(obj->obj, obj->mailbox);
3385
3386         /* Thread. */
3387         thread_ip_inc_cond(t, done);
3388         thread_yield_cond(p, done ^ 1);
3389 }
3390
3391 static inline void
3392 instr_extern_func_exec(struct rte_swx_pipeline *p)
3393 {
3394         struct thread *t = &p->threads[p->thread_id];
3395         struct instruction *ip = t->ip;
3396         uint32_t ext_func_id = ip->ext_func.ext_func_id;
3397         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3398         rte_swx_extern_func_t func = ext_func->func;
3399
3400         TRACE("[Thread %2u] extern func %u\n",
3401               p->thread_id,
3402               ext_func_id);
3403
3404         /* Extern function execute. */
3405         uint32_t done = func(ext_func->mailbox);
3406
3407         /* Thread. */
3408         thread_ip_inc_cond(t, done);
3409         thread_yield_cond(p, done ^ 1);
3410 }
3411
3412 /*
3413  * mov.
3414  */
3415 static int
3416 instr_mov_translate(struct rte_swx_pipeline *p,
3417                     struct action *action,
3418                     char **tokens,
3419                     int n_tokens,
3420                     struct instruction *instr,
3421                     struct instruction_data *data __rte_unused)
3422 {
3423         char *dst = tokens[1], *src = tokens[2];
3424         struct field *fdst, *fsrc;
3425         uint64_t src_val;
3426         uint32_t dst_struct_id, src_struct_id;
3427
3428         CHECK(n_tokens == 3, EINVAL);
3429
3430         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3431         CHECK(fdst, EINVAL);
3432
3433         /* MOV or MOV_S. */
3434         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3435         if (fsrc) {
3436                 instr->type = INSTR_MOV;
3437                 if ((dst[0] == 'h' && src[0] != 'h') ||
3438                     (dst[0] != 'h' && src[0] == 'h'))
3439                         instr->type = INSTR_MOV_S;
3440
3441                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3442                 instr->mov.dst.n_bits = fdst->n_bits;
3443                 instr->mov.dst.offset = fdst->offset / 8;
3444                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3445                 instr->mov.src.n_bits = fsrc->n_bits;
3446                 instr->mov.src.offset = fsrc->offset / 8;
3447                 return 0;
3448         }
3449
3450         /* MOV_I. */
3451         src_val = strtoull(src, &src, 0);
3452         CHECK(!src[0], EINVAL);
3453
3454         if (dst[0] == 'h')
3455                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3456
3457         instr->type = INSTR_MOV_I;
3458         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3459         instr->mov.dst.n_bits = fdst->n_bits;
3460         instr->mov.dst.offset = fdst->offset / 8;
3461         instr->mov.src_val = src_val;
3462         return 0;
3463 }
3464
3465 static inline void
3466 instr_mov_exec(struct rte_swx_pipeline *p)
3467 {
3468         struct thread *t = &p->threads[p->thread_id];
3469         struct instruction *ip = t->ip;
3470
3471         TRACE("[Thread %2u] mov\n",
3472               p->thread_id);
3473
3474         MOV(t, ip);
3475
3476         /* Thread. */
3477         thread_ip_inc(p);
3478 }
3479
3480 static inline void
3481 instr_mov_s_exec(struct rte_swx_pipeline *p)
3482 {
3483         struct thread *t = &p->threads[p->thread_id];
3484         struct instruction *ip = t->ip;
3485
3486         TRACE("[Thread %2u] mov (s)\n",
3487               p->thread_id);
3488
3489         MOV_S(t, ip);
3490
3491         /* Thread. */
3492         thread_ip_inc(p);
3493 }
3494
3495 static inline void
3496 instr_mov_i_exec(struct rte_swx_pipeline *p)
3497 {
3498         struct thread *t = &p->threads[p->thread_id];
3499         struct instruction *ip = t->ip;
3500
3501         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
3502               p->thread_id,
3503               ip->mov.src_val);
3504
3505         MOV_I(t, ip);
3506
3507         /* Thread. */
3508         thread_ip_inc(p);
3509 }
3510
3511 /*
3512  * dma.
3513  */
3514 static int
3515 instr_dma_translate(struct rte_swx_pipeline *p,
3516                     struct action *action,
3517                     char **tokens,
3518                     int n_tokens,
3519                     struct instruction *instr,
3520                     struct instruction_data *data __rte_unused)
3521 {
3522         char *dst = tokens[1];
3523         char *src = tokens[2];
3524         struct header *h;
3525         struct field *tf;
3526
3527         CHECK(action, EINVAL);
3528         CHECK(n_tokens == 3, EINVAL);
3529
3530         h = header_parse(p, dst);
3531         CHECK(h, EINVAL);
3532
3533         tf = action_field_parse(action, src);
3534         CHECK(tf, EINVAL);
3535
3536         instr->type = INSTR_DMA_HT;
3537         instr->dma.dst.header_id[0] = h->id;
3538         instr->dma.dst.struct_id[0] = h->struct_id;
3539         instr->dma.n_bytes[0] = h->st->n_bits / 8;
3540         instr->dma.src.offset[0] = tf->offset / 8;
3541
3542         return 0;
3543 }
3544
3545 static inline void
3546 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3547
3548 static inline void
3549 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3550 {
3551         struct thread *t = &p->threads[p->thread_id];
3552         struct instruction *ip = t->ip;
3553         uint8_t *action_data = t->structs[0];
3554         uint64_t valid_headers = t->valid_headers;
3555         uint32_t i;
3556
3557         for (i = 0; i < n_dma; i++) {
3558                 uint32_t header_id = ip->dma.dst.header_id[i];
3559                 uint32_t struct_id = ip->dma.dst.struct_id[i];
3560                 uint32_t offset = ip->dma.src.offset[i];
3561                 uint32_t n_bytes = ip->dma.n_bytes[i];
3562
3563                 struct header_runtime *h = &t->headers[header_id];
3564                 uint8_t *h_ptr0 = h->ptr0;
3565                 uint8_t *h_ptr = t->structs[struct_id];
3566
3567                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3568                         h_ptr : h_ptr0;
3569                 void *src = &action_data[offset];
3570
3571                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3572
3573                 /* Headers. */
3574                 memcpy(dst, src, n_bytes);
3575                 t->structs[struct_id] = dst;
3576                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3577         }
3578
3579         t->valid_headers = valid_headers;
3580 }
3581
3582 static inline void
3583 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3584 {
3585         __instr_dma_ht_exec(p, 1);
3586
3587         /* Thread. */
3588         thread_ip_inc(p);
3589 }
3590
3591 static inline void
3592 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3593 {
3594         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3595               p->thread_id);
3596
3597         __instr_dma_ht_exec(p, 2);
3598
3599         /* Thread. */
3600         thread_ip_inc(p);
3601 }
3602
3603 static inline void
3604 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3605 {
3606         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3607               p->thread_id);
3608
3609         __instr_dma_ht_exec(p, 3);
3610
3611         /* Thread. */
3612         thread_ip_inc(p);
3613 }
3614
3615 static inline void
3616 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3617 {
3618         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3619               p->thread_id);
3620
3621         __instr_dma_ht_exec(p, 4);
3622
3623         /* Thread. */
3624         thread_ip_inc(p);
3625 }
3626
3627 static inline void
3628 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3629 {
3630         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3631               p->thread_id);
3632
3633         __instr_dma_ht_exec(p, 5);
3634
3635         /* Thread. */
3636         thread_ip_inc(p);
3637 }
3638
3639 static inline void
3640 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3641 {
3642         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3643               p->thread_id);
3644
3645         __instr_dma_ht_exec(p, 6);
3646
3647         /* Thread. */
3648         thread_ip_inc(p);
3649 }
3650
3651 static inline void
3652 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3653 {
3654         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3655               p->thread_id);
3656
3657         __instr_dma_ht_exec(p, 7);
3658
3659         /* Thread. */
3660         thread_ip_inc(p);
3661 }
3662
3663 static inline void
3664 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3665 {
3666         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3667               p->thread_id);
3668
3669         __instr_dma_ht_exec(p, 8);
3670
3671         /* Thread. */
3672         thread_ip_inc(p);
3673 }
3674
3675 /*
3676  * alu.
3677  */
3678 static int
3679 instr_alu_add_translate(struct rte_swx_pipeline *p,
3680                         struct action *action,
3681                         char **tokens,
3682                         int n_tokens,
3683                         struct instruction *instr,
3684                         struct instruction_data *data __rte_unused)
3685 {
3686         char *dst = tokens[1], *src = tokens[2];
3687         struct field *fdst, *fsrc;
3688         uint64_t src_val;
3689         uint32_t dst_struct_id, src_struct_id;
3690
3691         CHECK(n_tokens == 3, EINVAL);
3692
3693         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3694         CHECK(fdst, EINVAL);
3695
3696         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3697         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3698         if (fsrc) {
3699                 instr->type = INSTR_ALU_ADD;
3700                 if (dst[0] == 'h' && src[0] != 'h')
3701                         instr->type = INSTR_ALU_ADD_HM;
3702                 if (dst[0] != 'h' && src[0] == 'h')
3703                         instr->type = INSTR_ALU_ADD_MH;
3704                 if (dst[0] == 'h' && src[0] == 'h')
3705                         instr->type = INSTR_ALU_ADD_HH;
3706
3707                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3708                 instr->alu.dst.n_bits = fdst->n_bits;
3709                 instr->alu.dst.offset = fdst->offset / 8;
3710                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3711                 instr->alu.src.n_bits = fsrc->n_bits;
3712                 instr->alu.src.offset = fsrc->offset / 8;
3713                 return 0;
3714         }
3715
3716         /* ADD_MI, ADD_HI. */
3717         src_val = strtoull(src, &src, 0);
3718         CHECK(!src[0], EINVAL);
3719
3720         instr->type = INSTR_ALU_ADD_MI;
3721         if (dst[0] == 'h')
3722                 instr->type = INSTR_ALU_ADD_HI;
3723
3724         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3725         instr->alu.dst.n_bits = fdst->n_bits;
3726         instr->alu.dst.offset = fdst->offset / 8;
3727         instr->alu.src_val = src_val;
3728         return 0;
3729 }
3730
3731 static int
3732 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3733                         struct action *action,
3734                         char **tokens,
3735                         int n_tokens,
3736                         struct instruction *instr,
3737                         struct instruction_data *data __rte_unused)
3738 {
3739         char *dst = tokens[1], *src = tokens[2];
3740         struct field *fdst, *fsrc;
3741         uint64_t src_val;
3742         uint32_t dst_struct_id, src_struct_id;
3743
3744         CHECK(n_tokens == 3, EINVAL);
3745
3746         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3747         CHECK(fdst, EINVAL);
3748
3749         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3750         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3751         if (fsrc) {
3752                 instr->type = INSTR_ALU_SUB;
3753                 if (dst[0] == 'h' && src[0] != 'h')
3754                         instr->type = INSTR_ALU_SUB_HM;
3755                 if (dst[0] != 'h' && src[0] == 'h')
3756                         instr->type = INSTR_ALU_SUB_MH;
3757                 if (dst[0] == 'h' && src[0] == 'h')
3758                         instr->type = INSTR_ALU_SUB_HH;
3759
3760                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3761                 instr->alu.dst.n_bits = fdst->n_bits;
3762                 instr->alu.dst.offset = fdst->offset / 8;
3763                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3764                 instr->alu.src.n_bits = fsrc->n_bits;
3765                 instr->alu.src.offset = fsrc->offset / 8;
3766                 return 0;
3767         }
3768
3769         /* SUB_MI, SUB_HI. */
3770         src_val = strtoull(src, &src, 0);
3771         CHECK(!src[0], EINVAL);
3772
3773         instr->type = INSTR_ALU_SUB_MI;
3774         if (dst[0] == 'h')
3775                 instr->type = INSTR_ALU_SUB_HI;
3776
3777         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3778         instr->alu.dst.n_bits = fdst->n_bits;
3779         instr->alu.dst.offset = fdst->offset / 8;
3780         instr->alu.src_val = src_val;
3781         return 0;
3782 }
3783
3784 static int
3785 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3786                           struct action *action __rte_unused,
3787                           char **tokens,
3788                           int n_tokens,
3789                           struct instruction *instr,
3790                           struct instruction_data *data __rte_unused)
3791 {
3792         char *dst = tokens[1], *src = tokens[2];
3793         struct header *hdst, *hsrc;
3794         struct field *fdst, *fsrc;
3795
3796         CHECK(n_tokens == 3, EINVAL);
3797
3798         fdst = header_field_parse(p, dst, &hdst);
3799         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3800
3801         /* CKADD_FIELD. */
3802         fsrc = header_field_parse(p, src, &hsrc);
3803         if (fsrc) {
3804                 instr->type = INSTR_ALU_CKADD_FIELD;
3805                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3806                 instr->alu.dst.n_bits = fdst->n_bits;
3807                 instr->alu.dst.offset = fdst->offset / 8;
3808                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3809                 instr->alu.src.n_bits = fsrc->n_bits;
3810                 instr->alu.src.offset = fsrc->offset / 8;
3811                 return 0;
3812         }
3813
3814         /* CKADD_STRUCT, CKADD_STRUCT20. */
3815         hsrc = header_parse(p, src);
3816         CHECK(hsrc, EINVAL);
3817
3818         instr->type = INSTR_ALU_CKADD_STRUCT;
3819         if ((hsrc->st->n_bits / 8) == 20)
3820                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3821
3822         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3823         instr->alu.dst.n_bits = fdst->n_bits;
3824         instr->alu.dst.offset = fdst->offset / 8;
3825         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3826         instr->alu.src.n_bits = hsrc->st->n_bits;
3827         instr->alu.src.offset = 0; /* Unused. */
3828         return 0;
3829 }
3830
3831 static int
3832 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3833                           struct action *action __rte_unused,
3834                           char **tokens,
3835                           int n_tokens,
3836                           struct instruction *instr,
3837                           struct instruction_data *data __rte_unused)
3838 {
3839         char *dst = tokens[1], *src = tokens[2];
3840         struct header *hdst, *hsrc;
3841         struct field *fdst, *fsrc;
3842
3843         CHECK(n_tokens == 3, EINVAL);
3844
3845         fdst = header_field_parse(p, dst, &hdst);
3846         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3847
3848         fsrc = header_field_parse(p, src, &hsrc);
3849         CHECK(fsrc, EINVAL);
3850
3851         instr->type = INSTR_ALU_CKSUB_FIELD;
3852         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3853         instr->alu.dst.n_bits = fdst->n_bits;
3854         instr->alu.dst.offset = fdst->offset / 8;
3855         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3856         instr->alu.src.n_bits = fsrc->n_bits;
3857         instr->alu.src.offset = fsrc->offset / 8;
3858         return 0;
3859 }
3860
3861 static int
3862 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3863                         struct action *action,
3864                         char **tokens,
3865                         int n_tokens,
3866                         struct instruction *instr,
3867                         struct instruction_data *data __rte_unused)
3868 {
3869         char *dst = tokens[1], *src = tokens[2];
3870         struct field *fdst, *fsrc;
3871         uint64_t src_val;
3872         uint32_t dst_struct_id, src_struct_id;
3873
3874         CHECK(n_tokens == 3, EINVAL);
3875
3876         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3877         CHECK(fdst, EINVAL);
3878
3879         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3880         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3881         if (fsrc) {
3882                 instr->type = INSTR_ALU_SHL;
3883                 if (dst[0] == 'h' && src[0] != 'h')
3884                         instr->type = INSTR_ALU_SHL_HM;
3885                 if (dst[0] != 'h' && src[0] == 'h')
3886                         instr->type = INSTR_ALU_SHL_MH;
3887                 if (dst[0] == 'h' && src[0] == 'h')
3888                         instr->type = INSTR_ALU_SHL_HH;
3889
3890                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3891                 instr->alu.dst.n_bits = fdst->n_bits;
3892                 instr->alu.dst.offset = fdst->offset / 8;
3893                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3894                 instr->alu.src.n_bits = fsrc->n_bits;
3895                 instr->alu.src.offset = fsrc->offset / 8;
3896                 return 0;
3897         }
3898
3899         /* SHL_MI, SHL_HI. */
3900         src_val = strtoull(src, &src, 0);
3901         CHECK(!src[0], EINVAL);
3902
3903         instr->type = INSTR_ALU_SHL_MI;
3904         if (dst[0] == 'h')
3905                 instr->type = INSTR_ALU_SHL_HI;
3906
3907         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3908         instr->alu.dst.n_bits = fdst->n_bits;
3909         instr->alu.dst.offset = fdst->offset / 8;
3910         instr->alu.src_val = src_val;
3911         return 0;
3912 }
3913
3914 static int
3915 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3916                         struct action *action,
3917                         char **tokens,
3918                         int n_tokens,
3919                         struct instruction *instr,
3920                         struct instruction_data *data __rte_unused)
3921 {
3922         char *dst = tokens[1], *src = tokens[2];
3923         struct field *fdst, *fsrc;
3924         uint64_t src_val;
3925         uint32_t dst_struct_id, src_struct_id;
3926
3927         CHECK(n_tokens == 3, EINVAL);
3928
3929         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3930         CHECK(fdst, EINVAL);
3931
3932         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3933         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3934         if (fsrc) {
3935                 instr->type = INSTR_ALU_SHR;
3936                 if (dst[0] == 'h' && src[0] != 'h')
3937                         instr->type = INSTR_ALU_SHR_HM;
3938                 if (dst[0] != 'h' && src[0] == 'h')
3939                         instr->type = INSTR_ALU_SHR_MH;
3940                 if (dst[0] == 'h' && src[0] == 'h')
3941                         instr->type = INSTR_ALU_SHR_HH;
3942
3943                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3944                 instr->alu.dst.n_bits = fdst->n_bits;
3945                 instr->alu.dst.offset = fdst->offset / 8;
3946                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3947                 instr->alu.src.n_bits = fsrc->n_bits;
3948                 instr->alu.src.offset = fsrc->offset / 8;
3949                 return 0;
3950         }
3951
3952         /* SHR_MI, SHR_HI. */
3953         src_val = strtoull(src, &src, 0);
3954         CHECK(!src[0], EINVAL);
3955
3956         instr->type = INSTR_ALU_SHR_MI;
3957         if (dst[0] == 'h')
3958                 instr->type = INSTR_ALU_SHR_HI;
3959
3960         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3961         instr->alu.dst.n_bits = fdst->n_bits;
3962         instr->alu.dst.offset = fdst->offset / 8;
3963         instr->alu.src_val = src_val;
3964         return 0;
3965 }
3966
3967 static int
3968 instr_alu_and_translate(struct rte_swx_pipeline *p,
3969                         struct action *action,
3970                         char **tokens,
3971                         int n_tokens,
3972                         struct instruction *instr,
3973                         struct instruction_data *data __rte_unused)
3974 {
3975         char *dst = tokens[1], *src = tokens[2];
3976         struct field *fdst, *fsrc;
3977         uint64_t src_val;
3978         uint32_t dst_struct_id, src_struct_id;
3979
3980         CHECK(n_tokens == 3, EINVAL);
3981
3982         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3983         CHECK(fdst, EINVAL);
3984
3985         /* AND or AND_S. */
3986         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3987         if (fsrc) {
3988                 instr->type = INSTR_ALU_AND;
3989                 if ((dst[0] == 'h' && src[0] != 'h') ||
3990                     (dst[0] != 'h' && src[0] == 'h'))
3991                         instr->type = INSTR_ALU_AND_S;
3992
3993                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3994                 instr->alu.dst.n_bits = fdst->n_bits;
3995                 instr->alu.dst.offset = fdst->offset / 8;
3996                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3997                 instr->alu.src.n_bits = fsrc->n_bits;
3998                 instr->alu.src.offset = fsrc->offset / 8;
3999                 return 0;
4000         }
4001
4002         /* AND_I. */
4003         src_val = strtoull(src, &src, 0);
4004         CHECK(!src[0], EINVAL);
4005
4006         if (dst[0] == 'h')
4007                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4008
4009         instr->type = INSTR_ALU_AND_I;
4010         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4011         instr->alu.dst.n_bits = fdst->n_bits;
4012         instr->alu.dst.offset = fdst->offset / 8;
4013         instr->alu.src_val = src_val;
4014         return 0;
4015 }
4016
4017 static int
4018 instr_alu_or_translate(struct rte_swx_pipeline *p,
4019                        struct action *action,
4020                        char **tokens,
4021                        int n_tokens,
4022                        struct instruction *instr,
4023                        struct instruction_data *data __rte_unused)
4024 {
4025         char *dst = tokens[1], *src = tokens[2];
4026         struct field *fdst, *fsrc;
4027         uint64_t src_val;
4028         uint32_t dst_struct_id, src_struct_id;
4029
4030         CHECK(n_tokens == 3, EINVAL);
4031
4032         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4033         CHECK(fdst, EINVAL);
4034
4035         /* OR or OR_S. */
4036         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4037         if (fsrc) {
4038                 instr->type = INSTR_ALU_OR;
4039                 if ((dst[0] == 'h' && src[0] != 'h') ||
4040                     (dst[0] != 'h' && src[0] == 'h'))
4041                         instr->type = INSTR_ALU_OR_S;
4042
4043                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4044                 instr->alu.dst.n_bits = fdst->n_bits;
4045                 instr->alu.dst.offset = fdst->offset / 8;
4046                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4047                 instr->alu.src.n_bits = fsrc->n_bits;
4048                 instr->alu.src.offset = fsrc->offset / 8;
4049                 return 0;
4050         }
4051
4052         /* OR_I. */
4053         src_val = strtoull(src, &src, 0);
4054         CHECK(!src[0], EINVAL);
4055
4056         if (dst[0] == 'h')
4057                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4058
4059         instr->type = INSTR_ALU_OR_I;
4060         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4061         instr->alu.dst.n_bits = fdst->n_bits;
4062         instr->alu.dst.offset = fdst->offset / 8;
4063         instr->alu.src_val = src_val;
4064         return 0;
4065 }
4066
4067 static int
4068 instr_alu_xor_translate(struct rte_swx_pipeline *p,
4069                         struct action *action,
4070                         char **tokens,
4071                         int n_tokens,
4072                         struct instruction *instr,
4073                         struct instruction_data *data __rte_unused)
4074 {
4075         char *dst = tokens[1], *src = tokens[2];
4076         struct field *fdst, *fsrc;
4077         uint64_t src_val;
4078         uint32_t dst_struct_id, src_struct_id;
4079
4080         CHECK(n_tokens == 3, EINVAL);
4081
4082         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4083         CHECK(fdst, EINVAL);
4084
4085         /* XOR or XOR_S. */
4086         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4087         if (fsrc) {
4088                 instr->type = INSTR_ALU_XOR;
4089                 if ((dst[0] == 'h' && src[0] != 'h') ||
4090                     (dst[0] != 'h' && src[0] == 'h'))
4091                         instr->type = INSTR_ALU_XOR_S;
4092
4093                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4094                 instr->alu.dst.n_bits = fdst->n_bits;
4095                 instr->alu.dst.offset = fdst->offset / 8;
4096                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4097                 instr->alu.src.n_bits = fsrc->n_bits;
4098                 instr->alu.src.offset = fsrc->offset / 8;
4099                 return 0;
4100         }
4101
4102         /* XOR_I. */
4103         src_val = strtoull(src, &src, 0);
4104         CHECK(!src[0], EINVAL);
4105
4106         if (dst[0] == 'h')
4107                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4108
4109         instr->type = INSTR_ALU_XOR_I;
4110         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4111         instr->alu.dst.n_bits = fdst->n_bits;
4112         instr->alu.dst.offset = fdst->offset / 8;
4113         instr->alu.src_val = src_val;
4114         return 0;
4115 }
4116
4117 static inline void
4118 instr_alu_add_exec(struct rte_swx_pipeline *p)
4119 {
4120         struct thread *t = &p->threads[p->thread_id];
4121         struct instruction *ip = t->ip;
4122
4123         TRACE("[Thread %2u] add\n", p->thread_id);
4124
4125         /* Structs. */
4126         ALU(t, ip, +);
4127
4128         /* Thread. */
4129         thread_ip_inc(p);
4130 }
4131
4132 static inline void
4133 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
4134 {
4135         struct thread *t = &p->threads[p->thread_id];
4136         struct instruction *ip = t->ip;
4137
4138         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
4139
4140         /* Structs. */
4141         ALU_MH(t, ip, +);
4142
4143         /* Thread. */
4144         thread_ip_inc(p);
4145 }
4146
4147 static inline void
4148 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
4149 {
4150         struct thread *t = &p->threads[p->thread_id];
4151         struct instruction *ip = t->ip;
4152
4153         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
4154
4155         /* Structs. */
4156         ALU_HM(t, ip, +);
4157
4158         /* Thread. */
4159         thread_ip_inc(p);
4160 }
4161
4162 static inline void
4163 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
4164 {
4165         struct thread *t = &p->threads[p->thread_id];
4166         struct instruction *ip = t->ip;
4167
4168         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
4169
4170         /* Structs. */
4171         ALU_HH(t, ip, +);
4172
4173         /* Thread. */
4174         thread_ip_inc(p);
4175 }
4176
4177 static inline void
4178 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
4179 {
4180         struct thread *t = &p->threads[p->thread_id];
4181         struct instruction *ip = t->ip;
4182
4183         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
4184
4185         /* Structs. */
4186         ALU_MI(t, ip, +);
4187
4188         /* Thread. */
4189         thread_ip_inc(p);
4190 }
4191
4192 static inline void
4193 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
4194 {
4195         struct thread *t = &p->threads[p->thread_id];
4196         struct instruction *ip = t->ip;
4197
4198         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
4199
4200         /* Structs. */
4201         ALU_HI(t, ip, +);
4202
4203         /* Thread. */
4204         thread_ip_inc(p);
4205 }
4206
4207 static inline void
4208 instr_alu_sub_exec(struct rte_swx_pipeline *p)
4209 {
4210         struct thread *t = &p->threads[p->thread_id];
4211         struct instruction *ip = t->ip;
4212
4213         TRACE("[Thread %2u] sub\n", p->thread_id);
4214
4215         /* Structs. */
4216         ALU(t, ip, -);
4217
4218         /* Thread. */
4219         thread_ip_inc(p);
4220 }
4221
4222 static inline void
4223 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
4224 {
4225         struct thread *t = &p->threads[p->thread_id];
4226         struct instruction *ip = t->ip;
4227
4228         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
4229
4230         /* Structs. */
4231         ALU_MH(t, ip, -);
4232
4233         /* Thread. */
4234         thread_ip_inc(p);
4235 }
4236
4237 static inline void
4238 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4239 {
4240         struct thread *t = &p->threads[p->thread_id];
4241         struct instruction *ip = t->ip;
4242
4243         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4244
4245         /* Structs. */
4246         ALU_HM(t, ip, -);
4247
4248         /* Thread. */
4249         thread_ip_inc(p);
4250 }
4251
4252 static inline void
4253 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4254 {
4255         struct thread *t = &p->threads[p->thread_id];
4256         struct instruction *ip = t->ip;
4257
4258         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4259
4260         /* Structs. */
4261         ALU_HH(t, ip, -);
4262
4263         /* Thread. */
4264         thread_ip_inc(p);
4265 }
4266
4267 static inline void
4268 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4269 {
4270         struct thread *t = &p->threads[p->thread_id];
4271         struct instruction *ip = t->ip;
4272
4273         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4274
4275         /* Structs. */
4276         ALU_MI(t, ip, -);
4277
4278         /* Thread. */
4279         thread_ip_inc(p);
4280 }
4281
4282 static inline void
4283 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4284 {
4285         struct thread *t = &p->threads[p->thread_id];
4286         struct instruction *ip = t->ip;
4287
4288         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4289
4290         /* Structs. */
4291         ALU_HI(t, ip, -);
4292
4293         /* Thread. */
4294         thread_ip_inc(p);
4295 }
4296
4297 static inline void
4298 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4299 {
4300         struct thread *t = &p->threads[p->thread_id];
4301         struct instruction *ip = t->ip;
4302
4303         TRACE("[Thread %2u] shl\n", p->thread_id);
4304
4305         /* Structs. */
4306         ALU(t, ip, <<);
4307
4308         /* Thread. */
4309         thread_ip_inc(p);
4310 }
4311
4312 static inline void
4313 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4314 {
4315         struct thread *t = &p->threads[p->thread_id];
4316         struct instruction *ip = t->ip;
4317
4318         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4319
4320         /* Structs. */
4321         ALU_MH(t, ip, <<);
4322
4323         /* Thread. */
4324         thread_ip_inc(p);
4325 }
4326
4327 static inline void
4328 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4329 {
4330         struct thread *t = &p->threads[p->thread_id];
4331         struct instruction *ip = t->ip;
4332
4333         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4334
4335         /* Structs. */
4336         ALU_HM(t, ip, <<);
4337
4338         /* Thread. */
4339         thread_ip_inc(p);
4340 }
4341
4342 static inline void
4343 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4344 {
4345         struct thread *t = &p->threads[p->thread_id];
4346         struct instruction *ip = t->ip;
4347
4348         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4349
4350         /* Structs. */
4351         ALU_HH(t, ip, <<);
4352
4353         /* Thread. */
4354         thread_ip_inc(p);
4355 }
4356
4357 static inline void
4358 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4359 {
4360         struct thread *t = &p->threads[p->thread_id];
4361         struct instruction *ip = t->ip;
4362
4363         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4364
4365         /* Structs. */
4366         ALU_MI(t, ip, <<);
4367
4368         /* Thread. */
4369         thread_ip_inc(p);
4370 }
4371
4372 static inline void
4373 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4374 {
4375         struct thread *t = &p->threads[p->thread_id];
4376         struct instruction *ip = t->ip;
4377
4378         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4379
4380         /* Structs. */
4381         ALU_HI(t, ip, <<);
4382
4383         /* Thread. */
4384         thread_ip_inc(p);
4385 }
4386
4387 static inline void
4388 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4389 {
4390         struct thread *t = &p->threads[p->thread_id];
4391         struct instruction *ip = t->ip;
4392
4393         TRACE("[Thread %2u] shr\n", p->thread_id);
4394
4395         /* Structs. */
4396         ALU(t, ip, >>);
4397
4398         /* Thread. */
4399         thread_ip_inc(p);
4400 }
4401
4402 static inline void
4403 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4404 {
4405         struct thread *t = &p->threads[p->thread_id];
4406         struct instruction *ip = t->ip;
4407
4408         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4409
4410         /* Structs. */
4411         ALU_MH(t, ip, >>);
4412
4413         /* Thread. */
4414         thread_ip_inc(p);
4415 }
4416
4417 static inline void
4418 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4419 {
4420         struct thread *t = &p->threads[p->thread_id];
4421         struct instruction *ip = t->ip;
4422
4423         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4424
4425         /* Structs. */
4426         ALU_HM(t, ip, >>);
4427
4428         /* Thread. */
4429         thread_ip_inc(p);
4430 }
4431
4432 static inline void
4433 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4434 {
4435         struct thread *t = &p->threads[p->thread_id];
4436         struct instruction *ip = t->ip;
4437
4438         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4439
4440         /* Structs. */
4441         ALU_HH(t, ip, >>);
4442
4443         /* Thread. */
4444         thread_ip_inc(p);
4445 }
4446
4447 static inline void
4448 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4449 {
4450         struct thread *t = &p->threads[p->thread_id];
4451         struct instruction *ip = t->ip;
4452
4453         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4454
4455         /* Structs. */
4456         ALU_MI(t, ip, >>);
4457
4458         /* Thread. */
4459         thread_ip_inc(p);
4460 }
4461
4462 static inline void
4463 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4464 {
4465         struct thread *t = &p->threads[p->thread_id];
4466         struct instruction *ip = t->ip;
4467
4468         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4469
4470         /* Structs. */
4471         ALU_HI(t, ip, >>);
4472
4473         /* Thread. */
4474         thread_ip_inc(p);
4475 }
4476
4477 static inline void
4478 instr_alu_and_exec(struct rte_swx_pipeline *p)
4479 {
4480         struct thread *t = &p->threads[p->thread_id];
4481         struct instruction *ip = t->ip;
4482
4483         TRACE("[Thread %2u] and\n", p->thread_id);
4484
4485         /* Structs. */
4486         ALU(t, ip, &);
4487
4488         /* Thread. */
4489         thread_ip_inc(p);
4490 }
4491
4492 static inline void
4493 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
4494 {
4495         struct thread *t = &p->threads[p->thread_id];
4496         struct instruction *ip = t->ip;
4497
4498         TRACE("[Thread %2u] and (s)\n", p->thread_id);
4499
4500         /* Structs. */
4501         ALU_S(t, ip, &);
4502
4503         /* Thread. */
4504         thread_ip_inc(p);
4505 }
4506
4507 static inline void
4508 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4509 {
4510         struct thread *t = &p->threads[p->thread_id];
4511         struct instruction *ip = t->ip;
4512
4513         TRACE("[Thread %2u] and (i)\n", p->thread_id);
4514
4515         /* Structs. */
4516         ALU_I(t, ip, &);
4517
4518         /* Thread. */
4519         thread_ip_inc(p);
4520 }
4521
4522 static inline void
4523 instr_alu_or_exec(struct rte_swx_pipeline *p)
4524 {
4525         struct thread *t = &p->threads[p->thread_id];
4526         struct instruction *ip = t->ip;
4527
4528         TRACE("[Thread %2u] or\n", p->thread_id);
4529
4530         /* Structs. */
4531         ALU(t, ip, |);
4532
4533         /* Thread. */
4534         thread_ip_inc(p);
4535 }
4536
4537 static inline void
4538 instr_alu_or_s_exec(struct rte_swx_pipeline *p)
4539 {
4540         struct thread *t = &p->threads[p->thread_id];
4541         struct instruction *ip = t->ip;
4542
4543         TRACE("[Thread %2u] or (s)\n", p->thread_id);
4544
4545         /* Structs. */
4546         ALU_S(t, ip, |);
4547
4548         /* Thread. */
4549         thread_ip_inc(p);
4550 }
4551
4552 static inline void
4553 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4554 {
4555         struct thread *t = &p->threads[p->thread_id];
4556         struct instruction *ip = t->ip;
4557
4558         TRACE("[Thread %2u] or (i)\n", p->thread_id);
4559
4560         /* Structs. */
4561         ALU_I(t, ip, |);
4562
4563         /* Thread. */
4564         thread_ip_inc(p);
4565 }
4566
4567 static inline void
4568 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4569 {
4570         struct thread *t = &p->threads[p->thread_id];
4571         struct instruction *ip = t->ip;
4572
4573         TRACE("[Thread %2u] xor\n", p->thread_id);
4574
4575         /* Structs. */
4576         ALU(t, ip, ^);
4577
4578         /* Thread. */
4579         thread_ip_inc(p);
4580 }
4581
4582 static inline void
4583 instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
4584 {
4585         struct thread *t = &p->threads[p->thread_id];
4586         struct instruction *ip = t->ip;
4587
4588         TRACE("[Thread %2u] xor (s)\n", p->thread_id);
4589
4590         /* Structs. */
4591         ALU_S(t, ip, ^);
4592
4593         /* Thread. */
4594         thread_ip_inc(p);
4595 }
4596
4597 static inline void
4598 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4599 {
4600         struct thread *t = &p->threads[p->thread_id];
4601         struct instruction *ip = t->ip;
4602
4603         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
4604
4605         /* Structs. */
4606         ALU_I(t, ip, ^);
4607
4608         /* Thread. */
4609         thread_ip_inc(p);
4610 }
4611
4612 static inline void
4613 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4614 {
4615         struct thread *t = &p->threads[p->thread_id];
4616         struct instruction *ip = t->ip;
4617         uint8_t *dst_struct, *src_struct;
4618         uint16_t *dst16_ptr, dst;
4619         uint64_t *src64_ptr, src64, src64_mask, src;
4620         uint64_t r;
4621
4622         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
4623
4624         /* Structs. */
4625         dst_struct = t->structs[ip->alu.dst.struct_id];
4626         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4627         dst = *dst16_ptr;
4628
4629         src_struct = t->structs[ip->alu.src.struct_id];
4630         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4631         src64 = *src64_ptr;
4632         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4633         src = src64 & src64_mask;
4634
4635         r = dst;
4636         r = ~r & 0xFFFF;
4637
4638         /* The first input (r) is a 16-bit number. The second and the third
4639          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
4640          * three numbers (output r) is a 34-bit number.
4641          */
4642         r += (src >> 32) + (src & 0xFFFFFFFF);
4643
4644         /* The first input is a 16-bit number. The second input is an 18-bit
4645          * number. In the worst case scenario, the sum of the two numbers is a
4646          * 19-bit number.
4647          */
4648         r = (r & 0xFFFF) + (r >> 16);
4649
4650         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4651          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
4652          */
4653         r = (r & 0xFFFF) + (r >> 16);
4654
4655         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4656          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4657          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
4658          * therefore the output r is always a 16-bit number.
4659          */
4660         r = (r & 0xFFFF) + (r >> 16);
4661
4662         r = ~r & 0xFFFF;
4663         r = r ? r : 0xFFFF;
4664
4665         *dst16_ptr = (uint16_t)r;
4666
4667         /* Thread. */
4668         thread_ip_inc(p);
4669 }
4670
4671 static inline void
4672 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4673 {
4674         struct thread *t = &p->threads[p->thread_id];
4675         struct instruction *ip = t->ip;
4676         uint8_t *dst_struct, *src_struct;
4677         uint16_t *dst16_ptr, dst;
4678         uint64_t *src64_ptr, src64, src64_mask, src;
4679         uint64_t r;
4680
4681         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
4682
4683         /* Structs. */
4684         dst_struct = t->structs[ip->alu.dst.struct_id];
4685         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4686         dst = *dst16_ptr;
4687
4688         src_struct = t->structs[ip->alu.src.struct_id];
4689         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4690         src64 = *src64_ptr;
4691         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4692         src = src64 & src64_mask;
4693
4694         r = dst;
4695         r = ~r & 0xFFFF;
4696
4697         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
4698          * the following sequence of operations in 2's complement arithmetic:
4699          *    a '- b = (a - b) % 0xFFFF.
4700          *
4701          * In order to prevent an underflow for the below subtraction, in which
4702          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
4703          * minuend), we first add a multiple of the 0xFFFF modulus to the
4704          * minuend. The number we add to the minuend needs to be a 34-bit number
4705          * or higher, so for readability reasons we picked the 36-bit multiple.
4706          * We are effectively turning the 16-bit minuend into a 36-bit number:
4707          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
4708          */
4709         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
4710
4711         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
4712          * result (the output r) is a 36-bit number.
4713          */
4714         r -= (src >> 32) + (src & 0xFFFFFFFF);
4715
4716         /* The first input is a 16-bit number. The second input is a 20-bit
4717          * number. Their sum is a 21-bit number.
4718          */
4719         r = (r & 0xFFFF) + (r >> 16);
4720
4721         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4722          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
4723          */
4724         r = (r & 0xFFFF) + (r >> 16);
4725
4726         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4727          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4728          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4729          * generated, therefore the output r is always a 16-bit number.
4730          */
4731         r = (r & 0xFFFF) + (r >> 16);
4732
4733         r = ~r & 0xFFFF;
4734         r = r ? r : 0xFFFF;
4735
4736         *dst16_ptr = (uint16_t)r;
4737
4738         /* Thread. */
4739         thread_ip_inc(p);
4740 }
4741
4742 static inline void
4743 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4744 {
4745         struct thread *t = &p->threads[p->thread_id];
4746         struct instruction *ip = t->ip;
4747         uint8_t *dst_struct, *src_struct;
4748         uint16_t *dst16_ptr;
4749         uint32_t *src32_ptr;
4750         uint64_t r0, r1;
4751
4752         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
4753
4754         /* Structs. */
4755         dst_struct = t->structs[ip->alu.dst.struct_id];
4756         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4757
4758         src_struct = t->structs[ip->alu.src.struct_id];
4759         src32_ptr = (uint32_t *)&src_struct[0];
4760
4761         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
4762         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
4763         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
4764         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
4765         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
4766
4767         /* The first input is a 16-bit number. The second input is a 19-bit
4768          * number. Their sum is a 20-bit number.
4769          */
4770         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4771
4772         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4773          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
4774          */
4775         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4776
4777         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4778          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4779          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
4780          * generated, therefore the output r is always a 16-bit number.
4781          */
4782         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4783
4784         r0 = ~r0 & 0xFFFF;
4785         r0 = r0 ? r0 : 0xFFFF;
4786
4787         *dst16_ptr = (uint16_t)r0;
4788
4789         /* Thread. */
4790         thread_ip_inc(p);
4791 }
4792
4793 static inline void
4794 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4795 {
4796         struct thread *t = &p->threads[p->thread_id];
4797         struct instruction *ip = t->ip;
4798         uint8_t *dst_struct, *src_struct;
4799         uint16_t *dst16_ptr;
4800         uint32_t *src32_ptr;
4801         uint64_t r = 0;
4802         uint32_t i;
4803
4804         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
4805
4806         /* Structs. */
4807         dst_struct = t->structs[ip->alu.dst.struct_id];
4808         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4809
4810         src_struct = t->structs[ip->alu.src.struct_id];
4811         src32_ptr = (uint32_t *)&src_struct[0];
4812
4813         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
4814          * Therefore, in the worst case scenario, a 35-bit number is added to a
4815          * 16-bit number (the input r), so the output r is 36-bit number.
4816          */
4817         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
4818                 r += *src32_ptr;
4819
4820         /* The first input is a 16-bit number. The second input is a 20-bit
4821          * number. Their sum is a 21-bit number.
4822          */
4823         r = (r & 0xFFFF) + (r >> 16);
4824
4825         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4826          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
4827          */
4828         r = (r & 0xFFFF) + (r >> 16);
4829
4830         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4831          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4832          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4833          * generated, therefore the output r is always a 16-bit number.
4834          */
4835         r = (r & 0xFFFF) + (r >> 16);
4836
4837         r = ~r & 0xFFFF;
4838         r = r ? r : 0xFFFF;
4839
4840         *dst16_ptr = (uint16_t)r;
4841
4842         /* Thread. */
4843         thread_ip_inc(p);
4844 }
4845
4846 /*
4847  * Register array.
4848  */
4849 static struct regarray *
4850 regarray_find(struct rte_swx_pipeline *p, const char *name);
4851
4852 static int
4853 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4854                       struct action *action,
4855                       char **tokens,
4856                       int n_tokens,
4857                       struct instruction *instr,
4858                       struct instruction_data *data __rte_unused)
4859 {
4860         char *regarray = tokens[1], *idx = tokens[2];
4861         struct regarray *r;
4862         struct field *fidx;
4863         uint32_t idx_struct_id, idx_val;
4864
4865         CHECK(n_tokens == 3, EINVAL);
4866
4867         r = regarray_find(p, regarray);
4868         CHECK(r, EINVAL);
4869
4870         /* REGPREFETCH_RH, REGPREFETCH_RM. */
4871         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4872         if (fidx) {
4873                 instr->type = INSTR_REGPREFETCH_RM;
4874                 if (idx[0] == 'h')
4875                         instr->type = INSTR_REGPREFETCH_RH;
4876
4877                 instr->regarray.regarray_id = r->id;
4878                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4879                 instr->regarray.idx.n_bits = fidx->n_bits;
4880                 instr->regarray.idx.offset = fidx->offset / 8;
4881                 instr->regarray.dstsrc_val = 0; /* Unused. */
4882                 return 0;
4883         }
4884
4885         /* REGPREFETCH_RI. */
4886         idx_val = strtoul(idx, &idx, 0);
4887         CHECK(!idx[0], EINVAL);
4888
4889         instr->type = INSTR_REGPREFETCH_RI;
4890         instr->regarray.regarray_id = r->id;
4891         instr->regarray.idx_val = idx_val;
4892         instr->regarray.dstsrc_val = 0; /* Unused. */
4893         return 0;
4894 }
4895
4896 static int
4897 instr_regrd_translate(struct rte_swx_pipeline *p,
4898                       struct action *action,
4899                       char **tokens,
4900                       int n_tokens,
4901                       struct instruction *instr,
4902                       struct instruction_data *data __rte_unused)
4903 {
4904         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4905         struct regarray *r;
4906         struct field *fdst, *fidx;
4907         uint32_t dst_struct_id, idx_struct_id, idx_val;
4908
4909         CHECK(n_tokens == 4, EINVAL);
4910
4911         r = regarray_find(p, regarray);
4912         CHECK(r, EINVAL);
4913
4914         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4915         CHECK(fdst, EINVAL);
4916
4917         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4918         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4919         if (fidx) {
4920                 instr->type = INSTR_REGRD_MRM;
4921                 if (dst[0] == 'h' && idx[0] != 'h')
4922                         instr->type = INSTR_REGRD_HRM;
4923                 if (dst[0] != 'h' && idx[0] == 'h')
4924                         instr->type = INSTR_REGRD_MRH;
4925                 if (dst[0] == 'h' && idx[0] == 'h')
4926                         instr->type = INSTR_REGRD_HRH;
4927
4928                 instr->regarray.regarray_id = r->id;
4929                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4930                 instr->regarray.idx.n_bits = fidx->n_bits;
4931                 instr->regarray.idx.offset = fidx->offset / 8;
4932                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4933                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4934                 instr->regarray.dstsrc.offset = fdst->offset / 8;
4935                 return 0;
4936         }
4937
4938         /* REGRD_MRI, REGRD_HRI. */
4939         idx_val = strtoul(idx, &idx, 0);
4940         CHECK(!idx[0], EINVAL);
4941
4942         instr->type = INSTR_REGRD_MRI;
4943         if (dst[0] == 'h')
4944                 instr->type = INSTR_REGRD_HRI;
4945
4946         instr->regarray.regarray_id = r->id;
4947         instr->regarray.idx_val = idx_val;
4948         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4949         instr->regarray.dstsrc.n_bits = fdst->n_bits;
4950         instr->regarray.dstsrc.offset = fdst->offset / 8;
4951         return 0;
4952 }
4953
4954 static int
4955 instr_regwr_translate(struct rte_swx_pipeline *p,
4956                       struct action *action,
4957                       char **tokens,
4958                       int n_tokens,
4959                       struct instruction *instr,
4960                       struct instruction_data *data __rte_unused)
4961 {
4962         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4963         struct regarray *r;
4964         struct field *fidx, *fsrc;
4965         uint64_t src_val;
4966         uint32_t idx_struct_id, idx_val, src_struct_id;
4967
4968         CHECK(n_tokens == 4, EINVAL);
4969
4970         r = regarray_find(p, regarray);
4971         CHECK(r, EINVAL);
4972
4973         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4974         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4975         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4976         if (fidx && fsrc) {
4977                 instr->type = INSTR_REGWR_RMM;
4978                 if (idx[0] == 'h' && src[0] != 'h')
4979                         instr->type = INSTR_REGWR_RHM;
4980                 if (idx[0] != 'h' && src[0] == 'h')
4981                         instr->type = INSTR_REGWR_RMH;
4982                 if (idx[0] == 'h' && src[0] == 'h')
4983                         instr->type = INSTR_REGWR_RHH;
4984
4985                 instr->regarray.regarray_id = r->id;
4986                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4987                 instr->regarray.idx.n_bits = fidx->n_bits;
4988                 instr->regarray.idx.offset = fidx->offset / 8;
4989                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4990                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4991                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4992                 return 0;
4993         }
4994
4995         /* REGWR_RHI, REGWR_RMI. */
4996         if (fidx && !fsrc) {
4997                 src_val = strtoull(src, &src, 0);
4998                 CHECK(!src[0], EINVAL);
4999
5000                 instr->type = INSTR_REGWR_RMI;
5001                 if (idx[0] == 'h')
5002                         instr->type = INSTR_REGWR_RHI;
5003
5004                 instr->regarray.regarray_id = r->id;
5005                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5006                 instr->regarray.idx.n_bits = fidx->n_bits;
5007                 instr->regarray.idx.offset = fidx->offset / 8;
5008                 instr->regarray.dstsrc_val = src_val;
5009                 return 0;
5010         }
5011
5012         /* REGWR_RIH, REGWR_RIM. */
5013         if (!fidx && fsrc) {
5014                 idx_val = strtoul(idx, &idx, 0);
5015                 CHECK(!idx[0], EINVAL);
5016
5017                 instr->type = INSTR_REGWR_RIM;
5018                 if (src[0] == 'h')
5019                         instr->type = INSTR_REGWR_RIH;
5020
5021                 instr->regarray.regarray_id = r->id;
5022                 instr->regarray.idx_val = idx_val;
5023                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5024                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5025                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5026                 return 0;
5027         }
5028
5029         /* REGWR_RII. */
5030         src_val = strtoull(src, &src, 0);
5031         CHECK(!src[0], EINVAL);
5032
5033         idx_val = strtoul(idx, &idx, 0);
5034         CHECK(!idx[0], EINVAL);
5035
5036         instr->type = INSTR_REGWR_RII;
5037         instr->regarray.idx_val = idx_val;
5038         instr->regarray.dstsrc_val = src_val;
5039
5040         return 0;
5041 }
5042
5043 static int
5044 instr_regadd_translate(struct rte_swx_pipeline *p,
5045                        struct action *action,
5046                        char **tokens,
5047                        int n_tokens,
5048                        struct instruction *instr,
5049                        struct instruction_data *data __rte_unused)
5050 {
5051         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5052         struct regarray *r;
5053         struct field *fidx, *fsrc;
5054         uint64_t src_val;
5055         uint32_t idx_struct_id, idx_val, src_struct_id;
5056
5057         CHECK(n_tokens == 4, EINVAL);
5058
5059         r = regarray_find(p, regarray);
5060         CHECK(r, EINVAL);
5061
5062         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
5063         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5064         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5065         if (fidx && fsrc) {
5066                 instr->type = INSTR_REGADD_RMM;
5067                 if (idx[0] == 'h' && src[0] != 'h')
5068                         instr->type = INSTR_REGADD_RHM;
5069                 if (idx[0] != 'h' && src[0] == 'h')
5070                         instr->type = INSTR_REGADD_RMH;
5071                 if (idx[0] == 'h' && src[0] == 'h')
5072                         instr->type = INSTR_REGADD_RHH;
5073
5074                 instr->regarray.regarray_id = r->id;
5075                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5076                 instr->regarray.idx.n_bits = fidx->n_bits;
5077                 instr->regarray.idx.offset = fidx->offset / 8;
5078                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5079                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5080                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5081                 return 0;
5082         }
5083
5084         /* REGADD_RHI, REGADD_RMI. */
5085         if (fidx && !fsrc) {
5086                 src_val = strtoull(src, &src, 0);
5087                 CHECK(!src[0], EINVAL);
5088
5089                 instr->type = INSTR_REGADD_RMI;
5090                 if (idx[0] == 'h')
5091                         instr->type = INSTR_REGADD_RHI;
5092
5093                 instr->regarray.regarray_id = r->id;
5094                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5095                 instr->regarray.idx.n_bits = fidx->n_bits;
5096                 instr->regarray.idx.offset = fidx->offset / 8;
5097                 instr->regarray.dstsrc_val = src_val;
5098                 return 0;
5099         }
5100
5101         /* REGADD_RIH, REGADD_RIM. */
5102         if (!fidx && fsrc) {
5103                 idx_val = strtoul(idx, &idx, 0);
5104                 CHECK(!idx[0], EINVAL);
5105
5106                 instr->type = INSTR_REGADD_RIM;
5107                 if (src[0] == 'h')
5108                         instr->type = INSTR_REGADD_RIH;
5109
5110                 instr->regarray.regarray_id = r->id;
5111                 instr->regarray.idx_val = idx_val;
5112                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5113                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5114                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5115                 return 0;
5116         }
5117
5118         /* REGADD_RII. */
5119         src_val = strtoull(src, &src, 0);
5120         CHECK(!src[0], EINVAL);
5121
5122         idx_val = strtoul(idx, &idx, 0);
5123         CHECK(!idx[0], EINVAL);
5124
5125         instr->type = INSTR_REGADD_RII;
5126         instr->regarray.idx_val = idx_val;
5127         instr->regarray.dstsrc_val = src_val;
5128         return 0;
5129 }
5130
5131 static inline uint64_t *
5132 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
5133 {
5134         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5135         return r->regarray;
5136 }
5137
5138 static inline uint64_t
5139 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5140 {
5141         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5142
5143         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5144         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5145         uint64_t idx64 = *idx64_ptr;
5146         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
5147         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5148
5149         return idx;
5150 }
5151
5152 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5153
5154 static inline uint64_t
5155 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5156 {
5157         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5158
5159         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5160         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5161         uint64_t idx64 = *idx64_ptr;
5162         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
5163
5164         return idx;
5165 }
5166
5167 #else
5168
5169 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
5170
5171 #endif
5172
5173 static inline uint64_t
5174 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5175 {
5176         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5177
5178         uint64_t idx = ip->regarray.idx_val & r->size_mask;
5179
5180         return idx;
5181 }
5182
5183 static inline uint64_t
5184 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
5185 {
5186         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5187         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5188         uint64_t src64 = *src64_ptr;
5189         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5190         uint64_t src = src64 & src64_mask;
5191
5192         return src;
5193 }
5194
5195 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5196
5197 static inline uint64_t
5198 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
5199 {
5200         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5201         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5202         uint64_t src64 = *src64_ptr;
5203         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
5204
5205         return src;
5206 }
5207
5208 #else
5209
5210 #define instr_regarray_src_nbo instr_regarray_src_hbo
5211
5212 #endif
5213
5214 static inline void
5215 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5216 {
5217         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5218         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5219         uint64_t dst64 = *dst64_ptr;
5220         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5221
5222         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5223
5224 }
5225
5226 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5227
5228 static inline void
5229 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5230 {
5231         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5232         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5233         uint64_t dst64 = *dst64_ptr;
5234         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5235
5236         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
5237         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5238 }
5239
5240 #else
5241
5242 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
5243
5244 #endif
5245
5246 static inline void
5247 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
5248 {
5249         struct thread *t = &p->threads[p->thread_id];
5250         struct instruction *ip = t->ip;
5251         uint64_t *regarray, idx;
5252
5253         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
5254
5255         /* Structs. */
5256         regarray = instr_regarray_regarray(p, ip);
5257         idx = instr_regarray_idx_nbo(p, t, ip);
5258         rte_prefetch0(&regarray[idx]);
5259
5260         /* Thread. */
5261         thread_ip_inc(p);
5262 }
5263
5264 static inline void
5265 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
5266 {
5267         struct thread *t = &p->threads[p->thread_id];
5268         struct instruction *ip = t->ip;
5269         uint64_t *regarray, idx;
5270
5271         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
5272
5273         /* Structs. */
5274         regarray = instr_regarray_regarray(p, ip);
5275         idx = instr_regarray_idx_hbo(p, t, ip);
5276         rte_prefetch0(&regarray[idx]);
5277
5278         /* Thread. */
5279         thread_ip_inc(p);
5280 }
5281
5282 static inline void
5283 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
5284 {
5285         struct thread *t = &p->threads[p->thread_id];
5286         struct instruction *ip = t->ip;
5287         uint64_t *regarray, idx;
5288
5289         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
5290
5291         /* Structs. */
5292         regarray = instr_regarray_regarray(p, ip);
5293         idx = instr_regarray_idx_imm(p, ip);
5294         rte_prefetch0(&regarray[idx]);
5295
5296         /* Thread. */
5297         thread_ip_inc(p);
5298 }
5299
5300 static inline void
5301 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
5302 {
5303         struct thread *t = &p->threads[p->thread_id];
5304         struct instruction *ip = t->ip;
5305         uint64_t *regarray, idx;
5306
5307         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
5308
5309         /* Structs. */
5310         regarray = instr_regarray_regarray(p, ip);
5311         idx = instr_regarray_idx_nbo(p, t, ip);
5312         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5313
5314         /* Thread. */
5315         thread_ip_inc(p);
5316 }
5317
5318 static inline void
5319 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
5320 {
5321         struct thread *t = &p->threads[p->thread_id];
5322         struct instruction *ip = t->ip;
5323         uint64_t *regarray, idx;
5324
5325         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
5326
5327         /* Structs. */
5328         regarray = instr_regarray_regarray(p, ip);
5329         idx = instr_regarray_idx_hbo(p, t, ip);
5330         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5331
5332         /* Thread. */
5333         thread_ip_inc(p);
5334 }
5335
5336 static inline void
5337 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
5338 {
5339         struct thread *t = &p->threads[p->thread_id];
5340         struct instruction *ip = t->ip;
5341         uint64_t *regarray, idx;
5342
5343         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
5344
5345         /* Structs. */
5346         regarray = instr_regarray_regarray(p, ip);
5347         idx = instr_regarray_idx_nbo(p, t, ip);
5348         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5349
5350         /* Thread. */
5351         thread_ip_inc(p);
5352 }
5353
5354 static inline void
5355 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
5356 {
5357         struct thread *t = &p->threads[p->thread_id];
5358         struct instruction *ip = t->ip;
5359         uint64_t *regarray, idx;
5360
5361         /* Structs. */
5362         regarray = instr_regarray_regarray(p, ip);
5363         idx = instr_regarray_idx_hbo(p, t, ip);
5364         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5365
5366         /* Thread. */
5367         thread_ip_inc(p);
5368 }
5369
5370 static inline void
5371 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
5372 {
5373         struct thread *t = &p->threads[p->thread_id];
5374         struct instruction *ip = t->ip;
5375         uint64_t *regarray, idx;
5376
5377         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
5378
5379         /* Structs. */
5380         regarray = instr_regarray_regarray(p, ip);
5381         idx = instr_regarray_idx_imm(p, ip);
5382         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5383
5384         /* Thread. */
5385         thread_ip_inc(p);
5386 }
5387
5388 static inline void
5389 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
5390 {
5391         struct thread *t = &p->threads[p->thread_id];
5392         struct instruction *ip = t->ip;
5393         uint64_t *regarray, idx;
5394
5395         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
5396
5397         /* Structs. */
5398         regarray = instr_regarray_regarray(p, ip);
5399         idx = instr_regarray_idx_imm(p, ip);
5400         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5401
5402         /* Thread. */
5403         thread_ip_inc(p);
5404 }
5405
5406 static inline void
5407 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
5408 {
5409         struct thread *t = &p->threads[p->thread_id];
5410         struct instruction *ip = t->ip;
5411         uint64_t *regarray, idx, src;
5412
5413         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
5414
5415         /* Structs. */
5416         regarray = instr_regarray_regarray(p, ip);
5417         idx = instr_regarray_idx_nbo(p, t, ip);
5418         src = instr_regarray_src_nbo(t, ip);
5419         regarray[idx] = src;
5420
5421         /* Thread. */
5422         thread_ip_inc(p);
5423 }
5424
5425 static inline void
5426 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
5427 {
5428         struct thread *t = &p->threads[p->thread_id];
5429         struct instruction *ip = t->ip;
5430         uint64_t *regarray, idx, src;
5431
5432         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
5433
5434         /* Structs. */
5435         regarray = instr_regarray_regarray(p, ip);
5436         idx = instr_regarray_idx_nbo(p, t, ip);
5437         src = instr_regarray_src_hbo(t, ip);
5438         regarray[idx] = src;
5439
5440         /* Thread. */
5441         thread_ip_inc(p);
5442 }
5443
5444 static inline void
5445 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
5446 {
5447         struct thread *t = &p->threads[p->thread_id];
5448         struct instruction *ip = t->ip;
5449         uint64_t *regarray, idx, src;
5450
5451         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
5452
5453         /* Structs. */
5454         regarray = instr_regarray_regarray(p, ip);
5455         idx = instr_regarray_idx_hbo(p, t, ip);
5456         src = instr_regarray_src_nbo(t, ip);
5457         regarray[idx] = src;
5458
5459         /* Thread. */
5460         thread_ip_inc(p);
5461 }
5462
5463 static inline void
5464 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
5465 {
5466         struct thread *t = &p->threads[p->thread_id];
5467         struct instruction *ip = t->ip;
5468         uint64_t *regarray, idx, src;
5469
5470         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
5471
5472         /* Structs. */
5473         regarray = instr_regarray_regarray(p, ip);
5474         idx = instr_regarray_idx_hbo(p, t, ip);
5475         src = instr_regarray_src_hbo(t, ip);
5476         regarray[idx] = src;
5477
5478         /* Thread. */
5479         thread_ip_inc(p);
5480 }
5481
5482 static inline void
5483 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
5484 {
5485         struct thread *t = &p->threads[p->thread_id];
5486         struct instruction *ip = t->ip;
5487         uint64_t *regarray, idx, src;
5488
5489         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
5490
5491         /* Structs. */
5492         regarray = instr_regarray_regarray(p, ip);
5493         idx = instr_regarray_idx_nbo(p, t, ip);
5494         src = ip->regarray.dstsrc_val;
5495         regarray[idx] = src;
5496
5497         /* Thread. */
5498         thread_ip_inc(p);
5499 }
5500
5501 static inline void
5502 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
5503 {
5504         struct thread *t = &p->threads[p->thread_id];
5505         struct instruction *ip = t->ip;
5506         uint64_t *regarray, idx, src;
5507
5508         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
5509
5510         /* Structs. */
5511         regarray = instr_regarray_regarray(p, ip);
5512         idx = instr_regarray_idx_hbo(p, t, ip);
5513         src = ip->regarray.dstsrc_val;
5514         regarray[idx] = src;
5515
5516         /* Thread. */
5517         thread_ip_inc(p);
5518 }
5519
5520 static inline void
5521 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
5522 {
5523         struct thread *t = &p->threads[p->thread_id];
5524         struct instruction *ip = t->ip;
5525         uint64_t *regarray, idx, src;
5526
5527         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
5528
5529         /* Structs. */
5530         regarray = instr_regarray_regarray(p, ip);
5531         idx = instr_regarray_idx_imm(p, ip);
5532         src = instr_regarray_src_nbo(t, ip);
5533         regarray[idx] = src;
5534
5535         /* Thread. */
5536         thread_ip_inc(p);
5537 }
5538
5539 static inline void
5540 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
5541 {
5542         struct thread *t = &p->threads[p->thread_id];
5543         struct instruction *ip = t->ip;
5544         uint64_t *regarray, idx, src;
5545
5546         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
5547
5548         /* Structs. */
5549         regarray = instr_regarray_regarray(p, ip);
5550         idx = instr_regarray_idx_imm(p, ip);
5551         src = instr_regarray_src_hbo(t, ip);
5552         regarray[idx] = src;
5553
5554         /* Thread. */
5555         thread_ip_inc(p);
5556 }
5557
5558 static inline void
5559 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
5560 {
5561         struct thread *t = &p->threads[p->thread_id];
5562         struct instruction *ip = t->ip;
5563         uint64_t *regarray, idx, src;
5564
5565         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
5566
5567         /* Structs. */
5568         regarray = instr_regarray_regarray(p, ip);
5569         idx = instr_regarray_idx_imm(p, ip);
5570         src = ip->regarray.dstsrc_val;
5571         regarray[idx] = src;
5572
5573         /* Thread. */
5574         thread_ip_inc(p);
5575 }
5576
5577 static inline void
5578 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
5579 {
5580         struct thread *t = &p->threads[p->thread_id];
5581         struct instruction *ip = t->ip;
5582         uint64_t *regarray, idx, src;
5583
5584         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
5585
5586         /* Structs. */
5587         regarray = instr_regarray_regarray(p, ip);
5588         idx = instr_regarray_idx_nbo(p, t, ip);
5589         src = instr_regarray_src_nbo(t, ip);
5590         regarray[idx] += src;
5591
5592         /* Thread. */
5593         thread_ip_inc(p);
5594 }
5595
5596 static inline void
5597 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
5598 {
5599         struct thread *t = &p->threads[p->thread_id];
5600         struct instruction *ip = t->ip;
5601         uint64_t *regarray, idx, src;
5602
5603         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
5604
5605         /* Structs. */
5606         regarray = instr_regarray_regarray(p, ip);
5607         idx = instr_regarray_idx_nbo(p, t, ip);
5608         src = instr_regarray_src_hbo(t, ip);
5609         regarray[idx] += src;
5610
5611         /* Thread. */
5612         thread_ip_inc(p);
5613 }
5614
5615 static inline void
5616 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
5617 {
5618         struct thread *t = &p->threads[p->thread_id];
5619         struct instruction *ip = t->ip;
5620         uint64_t *regarray, idx, src;
5621
5622         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
5623
5624         /* Structs. */
5625         regarray = instr_regarray_regarray(p, ip);
5626         idx = instr_regarray_idx_hbo(p, t, ip);
5627         src = instr_regarray_src_nbo(t, ip);
5628         regarray[idx] += src;
5629
5630         /* Thread. */
5631         thread_ip_inc(p);
5632 }
5633
5634 static inline void
5635 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
5636 {
5637         struct thread *t = &p->threads[p->thread_id];
5638         struct instruction *ip = t->ip;
5639         uint64_t *regarray, idx, src;
5640
5641         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
5642
5643         /* Structs. */
5644         regarray = instr_regarray_regarray(p, ip);
5645         idx = instr_regarray_idx_hbo(p, t, ip);
5646         src = instr_regarray_src_hbo(t, ip);
5647         regarray[idx] += src;
5648
5649         /* Thread. */
5650         thread_ip_inc(p);
5651 }
5652
5653 static inline void
5654 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
5655 {
5656         struct thread *t = &p->threads[p->thread_id];
5657         struct instruction *ip = t->ip;
5658         uint64_t *regarray, idx, src;
5659
5660         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
5661
5662         /* Structs. */
5663         regarray = instr_regarray_regarray(p, ip);
5664         idx = instr_regarray_idx_nbo(p, t, ip);
5665         src = ip->regarray.dstsrc_val;
5666         regarray[idx] += src;
5667
5668         /* Thread. */
5669         thread_ip_inc(p);
5670 }
5671
5672 static inline void
5673 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
5674 {
5675         struct thread *t = &p->threads[p->thread_id];
5676         struct instruction *ip = t->ip;
5677         uint64_t *regarray, idx, src;
5678
5679         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
5680
5681         /* Structs. */
5682         regarray = instr_regarray_regarray(p, ip);
5683         idx = instr_regarray_idx_hbo(p, t, ip);
5684         src = ip->regarray.dstsrc_val;
5685         regarray[idx] += src;
5686
5687         /* Thread. */
5688         thread_ip_inc(p);
5689 }
5690
5691 static inline void
5692 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
5693 {
5694         struct thread *t = &p->threads[p->thread_id];
5695         struct instruction *ip = t->ip;
5696         uint64_t *regarray, idx, src;
5697
5698         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
5699
5700         /* Structs. */
5701         regarray = instr_regarray_regarray(p, ip);
5702         idx = instr_regarray_idx_imm(p, ip);
5703         src = instr_regarray_src_nbo(t, ip);
5704         regarray[idx] += src;
5705
5706         /* Thread. */
5707         thread_ip_inc(p);
5708 }
5709
5710 static inline void
5711 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
5712 {
5713         struct thread *t = &p->threads[p->thread_id];
5714         struct instruction *ip = t->ip;
5715         uint64_t *regarray, idx, src;
5716
5717         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
5718
5719         /* Structs. */
5720         regarray = instr_regarray_regarray(p, ip);
5721         idx = instr_regarray_idx_imm(p, ip);
5722         src = instr_regarray_src_hbo(t, ip);
5723         regarray[idx] += src;
5724
5725         /* Thread. */
5726         thread_ip_inc(p);
5727 }
5728
5729 static inline void
5730 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
5731 {
5732         struct thread *t = &p->threads[p->thread_id];
5733         struct instruction *ip = t->ip;
5734         uint64_t *regarray, idx, src;
5735
5736         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
5737
5738         /* Structs. */
5739         regarray = instr_regarray_regarray(p, ip);
5740         idx = instr_regarray_idx_imm(p, ip);
5741         src = ip->regarray.dstsrc_val;
5742         regarray[idx] += src;
5743
5744         /* Thread. */
5745         thread_ip_inc(p);
5746 }
5747
5748 /*
5749  * metarray.
5750  */
5751 static struct metarray *
5752 metarray_find(struct rte_swx_pipeline *p, const char *name);
5753
5754 static int
5755 instr_metprefetch_translate(struct rte_swx_pipeline *p,
5756                             struct action *action,
5757                             char **tokens,
5758                             int n_tokens,
5759                             struct instruction *instr,
5760                             struct instruction_data *data __rte_unused)
5761 {
5762         char *metarray = tokens[1], *idx = tokens[2];
5763         struct metarray *m;
5764         struct field *fidx;
5765         uint32_t idx_struct_id, idx_val;
5766
5767         CHECK(n_tokens == 3, EINVAL);
5768
5769         m = metarray_find(p, metarray);
5770         CHECK(m, EINVAL);
5771
5772         /* METPREFETCH_H, METPREFETCH_M. */
5773         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5774         if (fidx) {
5775                 instr->type = INSTR_METPREFETCH_M;
5776                 if (idx[0] == 'h')
5777                         instr->type = INSTR_METPREFETCH_H;
5778
5779                 instr->meter.metarray_id = m->id;
5780                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5781                 instr->meter.idx.n_bits = fidx->n_bits;
5782                 instr->meter.idx.offset = fidx->offset / 8;
5783                 return 0;
5784         }
5785
5786         /* METPREFETCH_I. */
5787         idx_val = strtoul(idx, &idx, 0);
5788         CHECK(!idx[0], EINVAL);
5789
5790         instr->type = INSTR_METPREFETCH_I;
5791         instr->meter.metarray_id = m->id;
5792         instr->meter.idx_val = idx_val;
5793         return 0;
5794 }
5795
5796 static int
5797 instr_meter_translate(struct rte_swx_pipeline *p,
5798                       struct action *action,
5799                       char **tokens,
5800                       int n_tokens,
5801                       struct instruction *instr,
5802                       struct instruction_data *data __rte_unused)
5803 {
5804         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
5805         char *color_in = tokens[4], *color_out = tokens[5];
5806         struct metarray *m;
5807         struct field *fidx, *flength, *fcin, *fcout;
5808         uint32_t idx_struct_id, length_struct_id;
5809         uint32_t color_in_struct_id, color_out_struct_id;
5810
5811         CHECK(n_tokens == 6, EINVAL);
5812
5813         m = metarray_find(p, metarray);
5814         CHECK(m, EINVAL);
5815
5816         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5817
5818         flength = struct_field_parse(p, action, length, &length_struct_id);
5819         CHECK(flength, EINVAL);
5820
5821         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
5822
5823         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
5824         CHECK(fcout, EINVAL);
5825
5826         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
5827         if (fidx && fcin) {
5828                 instr->type = INSTR_METER_MMM;
5829                 if (idx[0] == 'h' && length[0] == 'h')
5830                         instr->type = INSTR_METER_HHM;
5831                 if (idx[0] == 'h' && length[0] != 'h')
5832                         instr->type = INSTR_METER_HMM;
5833                 if (idx[0] != 'h' && length[0] == 'h')
5834                         instr->type = INSTR_METER_MHM;
5835
5836                 instr->meter.metarray_id = m->id;
5837
5838                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5839                 instr->meter.idx.n_bits = fidx->n_bits;
5840                 instr->meter.idx.offset = fidx->offset / 8;
5841
5842                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5843                 instr->meter.length.n_bits = flength->n_bits;
5844                 instr->meter.length.offset = flength->offset / 8;
5845
5846                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5847                 instr->meter.color_in.n_bits = fcin->n_bits;
5848                 instr->meter.color_in.offset = fcin->offset / 8;
5849
5850                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5851                 instr->meter.color_out.n_bits = fcout->n_bits;
5852                 instr->meter.color_out.offset = fcout->offset / 8;
5853
5854                 return 0;
5855         }
5856
5857         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
5858         if (fidx && !fcin) {
5859                 uint32_t color_in_val = strtoul(color_in, &color_in, 0);
5860                 CHECK(!color_in[0], EINVAL);
5861
5862                 instr->type = INSTR_METER_MMI;
5863                 if (idx[0] == 'h' && length[0] == 'h')
5864                         instr->type = INSTR_METER_HHI;
5865                 if (idx[0] == 'h' && length[0] != 'h')
5866                         instr->type = INSTR_METER_HMI;
5867                 if (idx[0] != 'h' && length[0] == 'h')
5868                         instr->type = INSTR_METER_MHI;
5869
5870                 instr->meter.metarray_id = m->id;
5871
5872                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5873                 instr->meter.idx.n_bits = fidx->n_bits;
5874                 instr->meter.idx.offset = fidx->offset / 8;
5875
5876                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5877                 instr->meter.length.n_bits = flength->n_bits;
5878                 instr->meter.length.offset = flength->offset / 8;
5879
5880                 instr->meter.color_in_val = color_in_val;
5881
5882                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5883                 instr->meter.color_out.n_bits = fcout->n_bits;
5884                 instr->meter.color_out.offset = fcout->offset / 8;
5885
5886                 return 0;
5887         }
5888
5889         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5890         if (!fidx && fcin) {
5891                 uint32_t idx_val;
5892
5893                 idx_val = strtoul(idx, &idx, 0);
5894                 CHECK(!idx[0], EINVAL);
5895
5896                 instr->type = INSTR_METER_IMM;
5897                 if (length[0] == 'h')
5898                         instr->type = INSTR_METER_IHM;
5899
5900                 instr->meter.metarray_id = m->id;
5901
5902                 instr->meter.idx_val = idx_val;
5903
5904                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5905                 instr->meter.length.n_bits = flength->n_bits;
5906                 instr->meter.length.offset = flength->offset / 8;
5907
5908                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5909                 instr->meter.color_in.n_bits = fcin->n_bits;
5910                 instr->meter.color_in.offset = fcin->offset / 8;
5911
5912                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5913                 instr->meter.color_out.n_bits = fcout->n_bits;
5914                 instr->meter.color_out.offset = fcout->offset / 8;
5915
5916                 return 0;
5917         }
5918
5919         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5920         if (!fidx && !fcin) {
5921                 uint32_t idx_val, color_in_val;
5922
5923                 idx_val = strtoul(idx, &idx, 0);
5924                 CHECK(!idx[0], EINVAL);
5925
5926                 color_in_val = strtoul(color_in, &color_in, 0);
5927                 CHECK(!color_in[0], EINVAL);
5928
5929                 instr->type = INSTR_METER_IMI;
5930                 if (length[0] == 'h')
5931                         instr->type = INSTR_METER_IHI;
5932
5933                 instr->meter.metarray_id = m->id;
5934
5935                 instr->meter.idx_val = idx_val;
5936
5937                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5938                 instr->meter.length.n_bits = flength->n_bits;
5939                 instr->meter.length.offset = flength->offset / 8;
5940
5941                 instr->meter.color_in_val = color_in_val;
5942
5943                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5944                 instr->meter.color_out.n_bits = fcout->n_bits;
5945                 instr->meter.color_out.offset = fcout->offset / 8;
5946
5947                 return 0;
5948         }
5949
5950         CHECK(0, EINVAL);
5951 }
5952
5953 static inline struct meter *
5954 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5955 {
5956         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5957
5958         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5959         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5960         uint64_t idx64 = *idx64_ptr;
5961         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
5962         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5963
5964         return &r->metarray[idx];
5965 }
5966
5967 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5968
5969 static inline struct meter *
5970 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5971 {
5972         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5973
5974         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5975         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5976         uint64_t idx64 = *idx64_ptr;
5977         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
5978
5979         return &r->metarray[idx];
5980 }
5981
5982 #else
5983
5984 #define instr_meter_idx_nbo instr_meter_idx_hbo
5985
5986 #endif
5987
5988 static inline struct meter *
5989 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5990 {
5991         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5992
5993         uint64_t idx =  ip->meter.idx_val & r->size_mask;
5994
5995         return &r->metarray[idx];
5996 }
5997
5998 static inline uint32_t
5999 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
6000 {
6001         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6002         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6003         uint64_t src64 = *src64_ptr;
6004         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
6005         uint64_t src = src64 & src64_mask;
6006
6007         return (uint32_t)src;
6008 }
6009
6010 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6011
6012 static inline uint32_t
6013 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
6014 {
6015         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6016         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6017         uint64_t src64 = *src64_ptr;
6018         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
6019
6020         return (uint32_t)src;
6021 }
6022
6023 #else
6024
6025 #define instr_meter_length_nbo instr_meter_length_hbo
6026
6027 #endif
6028
6029 static inline enum rte_color
6030 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
6031 {
6032         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
6033         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
6034         uint64_t src64 = *src64_ptr;
6035         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
6036         uint64_t src = src64 & src64_mask;
6037
6038         return (enum rte_color)src;
6039 }
6040
6041 static inline void
6042 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
6043 {
6044         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
6045         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
6046         uint64_t dst64 = *dst64_ptr;
6047         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
6048
6049         uint64_t src = (uint64_t)color_out;
6050
6051         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6052 }
6053
6054 static inline void
6055 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
6056 {
6057         struct thread *t = &p->threads[p->thread_id];
6058         struct instruction *ip = t->ip;
6059         struct meter *m;
6060
6061         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
6062
6063         /* Structs. */
6064         m = instr_meter_idx_nbo(p, t, ip);
6065         rte_prefetch0(m);
6066
6067         /* Thread. */
6068         thread_ip_inc(p);
6069 }
6070
6071 static inline void
6072 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
6073 {
6074         struct thread *t = &p->threads[p->thread_id];
6075         struct instruction *ip = t->ip;
6076         struct meter *m;
6077
6078         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
6079
6080         /* Structs. */
6081         m = instr_meter_idx_hbo(p, t, ip);
6082         rte_prefetch0(m);
6083
6084         /* Thread. */
6085         thread_ip_inc(p);
6086 }
6087
6088 static inline void
6089 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
6090 {
6091         struct thread *t = &p->threads[p->thread_id];
6092         struct instruction *ip = t->ip;
6093         struct meter *m;
6094
6095         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
6096
6097         /* Structs. */
6098         m = instr_meter_idx_imm(p, ip);
6099         rte_prefetch0(m);
6100
6101         /* Thread. */
6102         thread_ip_inc(p);
6103 }
6104
6105 static inline void
6106 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
6107 {
6108         struct thread *t = &p->threads[p->thread_id];
6109         struct instruction *ip = t->ip;
6110         struct meter *m;
6111         uint64_t time, n_pkts, n_bytes;
6112         uint32_t length;
6113         enum rte_color color_in, color_out;
6114
6115         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
6116
6117         /* Structs. */
6118         m = instr_meter_idx_nbo(p, t, ip);
6119         rte_prefetch0(m->n_pkts);
6120         time = rte_get_tsc_cycles();
6121         length = instr_meter_length_nbo(t, ip);
6122         color_in = instr_meter_color_in_hbo(t, ip);
6123
6124         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6125                 &m->profile->profile,
6126                 time,
6127                 length,
6128                 color_in);
6129
6130         color_out &= m->color_mask;
6131
6132         n_pkts = m->n_pkts[color_out];
6133         n_bytes = m->n_bytes[color_out];
6134
6135         instr_meter_color_out_hbo_set(t, ip, color_out);
6136
6137         m->n_pkts[color_out] = n_pkts + 1;
6138         m->n_bytes[color_out] = n_bytes + length;
6139
6140         /* Thread. */
6141         thread_ip_inc(p);
6142 }
6143
6144 static inline void
6145 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
6146 {
6147         struct thread *t = &p->threads[p->thread_id];
6148         struct instruction *ip = t->ip;
6149         struct meter *m;
6150         uint64_t time, n_pkts, n_bytes;
6151         uint32_t length;
6152         enum rte_color color_in, color_out;
6153
6154         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
6155
6156         /* Structs. */
6157         m = instr_meter_idx_nbo(p, t, ip);
6158         rte_prefetch0(m->n_pkts);
6159         time = rte_get_tsc_cycles();
6160         length = instr_meter_length_nbo(t, ip);
6161         color_in = (enum rte_color)ip->meter.color_in_val;
6162
6163         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6164                 &m->profile->profile,
6165                 time,
6166                 length,
6167                 color_in);
6168
6169         color_out &= m->color_mask;
6170
6171         n_pkts = m->n_pkts[color_out];
6172         n_bytes = m->n_bytes[color_out];
6173
6174         instr_meter_color_out_hbo_set(t, ip, color_out);
6175
6176         m->n_pkts[color_out] = n_pkts + 1;
6177         m->n_bytes[color_out] = n_bytes + length;
6178
6179         /* Thread. */
6180         thread_ip_inc(p);
6181 }
6182
6183 static inline void
6184 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
6185 {
6186         struct thread *t = &p->threads[p->thread_id];
6187         struct instruction *ip = t->ip;
6188         struct meter *m;
6189         uint64_t time, n_pkts, n_bytes;
6190         uint32_t length;
6191         enum rte_color color_in, color_out;
6192
6193         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
6194
6195         /* Structs. */
6196         m = instr_meter_idx_nbo(p, t, ip);
6197         rte_prefetch0(m->n_pkts);
6198         time = rte_get_tsc_cycles();
6199         length = instr_meter_length_hbo(t, ip);
6200         color_in = instr_meter_color_in_hbo(t, ip);
6201
6202         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6203                 &m->profile->profile,
6204                 time,
6205                 length,
6206                 color_in);
6207
6208         color_out &= m->color_mask;
6209
6210         n_pkts = m->n_pkts[color_out];
6211         n_bytes = m->n_bytes[color_out];
6212
6213         instr_meter_color_out_hbo_set(t, ip, color_out);
6214
6215         m->n_pkts[color_out] = n_pkts + 1;
6216         m->n_bytes[color_out] = n_bytes + length;
6217
6218         /* Thread. */
6219         thread_ip_inc(p);
6220 }
6221 static inline void
6222 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
6223 {
6224         struct thread *t = &p->threads[p->thread_id];
6225         struct instruction *ip = t->ip;
6226         struct meter *m;
6227         uint64_t time, n_pkts, n_bytes;
6228         uint32_t length;
6229         enum rte_color color_in, color_out;
6230
6231         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
6232
6233         /* Structs. */
6234         m = instr_meter_idx_nbo(p, t, ip);
6235         rte_prefetch0(m->n_pkts);
6236         time = rte_get_tsc_cycles();
6237         length = instr_meter_length_hbo(t, ip);
6238         color_in = (enum rte_color)ip->meter.color_in_val;
6239
6240         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6241                 &m->profile->profile,
6242                 time,
6243                 length,
6244                 color_in);
6245
6246         color_out &= m->color_mask;
6247
6248         n_pkts = m->n_pkts[color_out];
6249         n_bytes = m->n_bytes[color_out];
6250
6251         instr_meter_color_out_hbo_set(t, ip, color_out);
6252
6253         m->n_pkts[color_out] = n_pkts + 1;
6254         m->n_bytes[color_out] = n_bytes + length;
6255
6256         /* Thread. */
6257         thread_ip_inc(p);
6258 }
6259
6260 static inline void
6261 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
6262 {
6263         struct thread *t = &p->threads[p->thread_id];
6264         struct instruction *ip = t->ip;
6265         struct meter *m;
6266         uint64_t time, n_pkts, n_bytes;
6267         uint32_t length;
6268         enum rte_color color_in, color_out;
6269
6270         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
6271
6272         /* Structs. */
6273         m = instr_meter_idx_hbo(p, t, ip);
6274         rte_prefetch0(m->n_pkts);
6275         time = rte_get_tsc_cycles();
6276         length = instr_meter_length_nbo(t, ip);
6277         color_in = instr_meter_color_in_hbo(t, ip);
6278
6279         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6280                 &m->profile->profile,
6281                 time,
6282                 length,
6283                 color_in);
6284
6285         color_out &= m->color_mask;
6286
6287         n_pkts = m->n_pkts[color_out];
6288         n_bytes = m->n_bytes[color_out];
6289
6290         instr_meter_color_out_hbo_set(t, ip, color_out);
6291
6292         m->n_pkts[color_out] = n_pkts + 1;
6293         m->n_bytes[color_out] = n_bytes + length;
6294
6295         /* Thread. */
6296         thread_ip_inc(p);
6297 }
6298
6299 static inline void
6300 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
6301 {
6302         struct thread *t = &p->threads[p->thread_id];
6303         struct instruction *ip = t->ip;
6304         struct meter *m;
6305         uint64_t time, n_pkts, n_bytes;
6306         uint32_t length;
6307         enum rte_color color_in, color_out;
6308
6309         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
6310
6311         /* Structs. */
6312         m = instr_meter_idx_hbo(p, t, ip);
6313         rte_prefetch0(m->n_pkts);
6314         time = rte_get_tsc_cycles();
6315         length = instr_meter_length_nbo(t, ip);
6316         color_in = (enum rte_color)ip->meter.color_in_val;
6317
6318         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6319                 &m->profile->profile,
6320                 time,
6321                 length,
6322                 color_in);
6323
6324         color_out &= m->color_mask;
6325
6326         n_pkts = m->n_pkts[color_out];
6327         n_bytes = m->n_bytes[color_out];
6328
6329         instr_meter_color_out_hbo_set(t, ip, color_out);
6330
6331         m->n_pkts[color_out] = n_pkts + 1;
6332         m->n_bytes[color_out] = n_bytes + length;
6333
6334         /* Thread. */
6335         thread_ip_inc(p);
6336 }
6337
6338 static inline void
6339 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
6340 {
6341         struct thread *t = &p->threads[p->thread_id];
6342         struct instruction *ip = t->ip;
6343         struct meter *m;
6344         uint64_t time, n_pkts, n_bytes;
6345         uint32_t length;
6346         enum rte_color color_in, color_out;
6347
6348         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
6349
6350         /* Structs. */
6351         m = instr_meter_idx_hbo(p, t, ip);
6352         rte_prefetch0(m->n_pkts);
6353         time = rte_get_tsc_cycles();
6354         length = instr_meter_length_hbo(t, ip);
6355         color_in = instr_meter_color_in_hbo(t, ip);
6356
6357         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6358                 &m->profile->profile,
6359                 time,
6360                 length,
6361                 color_in);
6362
6363         color_out &= m->color_mask;
6364
6365         n_pkts = m->n_pkts[color_out];
6366         n_bytes = m->n_bytes[color_out];
6367
6368         instr_meter_color_out_hbo_set(t, ip, color_out);
6369
6370         m->n_pkts[color_out] = n_pkts + 1;
6371         m->n_bytes[color_out] = n_bytes + length;
6372
6373         /* Thread. */
6374         thread_ip_inc(p);
6375 }
6376
6377 static inline void
6378 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
6379 {
6380         struct thread *t = &p->threads[p->thread_id];
6381         struct instruction *ip = t->ip;
6382         struct meter *m;
6383         uint64_t time, n_pkts, n_bytes;
6384         uint32_t length;
6385         enum rte_color color_in, color_out;
6386
6387         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
6388
6389         /* Structs. */
6390         m = instr_meter_idx_hbo(p, t, ip);
6391         rte_prefetch0(m->n_pkts);
6392         time = rte_get_tsc_cycles();
6393         length = instr_meter_length_hbo(t, ip);
6394         color_in = (enum rte_color)ip->meter.color_in_val;
6395
6396         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6397                 &m->profile->profile,
6398                 time,
6399                 length,
6400                 color_in);
6401
6402         color_out &= m->color_mask;
6403
6404         n_pkts = m->n_pkts[color_out];
6405         n_bytes = m->n_bytes[color_out];
6406
6407         instr_meter_color_out_hbo_set(t, ip, color_out);
6408
6409         m->n_pkts[color_out] = n_pkts + 1;
6410         m->n_bytes[color_out] = n_bytes + length;
6411
6412         /* Thread. */
6413         thread_ip_inc(p);
6414 }
6415
6416 static inline void
6417 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
6418 {
6419         struct thread *t = &p->threads[p->thread_id];
6420         struct instruction *ip = t->ip;
6421         struct meter *m;
6422         uint64_t time, n_pkts, n_bytes;
6423         uint32_t length;
6424         enum rte_color color_in, color_out;
6425
6426         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
6427
6428         /* Structs. */
6429         m = instr_meter_idx_imm(p, ip);
6430         rte_prefetch0(m->n_pkts);
6431         time = rte_get_tsc_cycles();
6432         length = instr_meter_length_nbo(t, ip);
6433         color_in = instr_meter_color_in_hbo(t, ip);
6434
6435         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6436                 &m->profile->profile,
6437                 time,
6438                 length,
6439                 color_in);
6440
6441         color_out &= m->color_mask;
6442
6443         n_pkts = m->n_pkts[color_out];
6444         n_bytes = m->n_bytes[color_out];
6445
6446         instr_meter_color_out_hbo_set(t, ip, color_out);
6447
6448         m->n_pkts[color_out] = n_pkts + 1;
6449         m->n_bytes[color_out] = n_bytes + length;
6450
6451         /* Thread. */
6452         thread_ip_inc(p);
6453 }
6454
6455 static inline void
6456 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
6457 {
6458         struct thread *t = &p->threads[p->thread_id];
6459         struct instruction *ip = t->ip;
6460         struct meter *m;
6461         uint64_t time, n_pkts, n_bytes;
6462         uint32_t length;
6463         enum rte_color color_in, color_out;
6464
6465         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
6466
6467         /* Structs. */
6468         m = instr_meter_idx_imm(p, ip);
6469         rte_prefetch0(m->n_pkts);
6470         time = rte_get_tsc_cycles();
6471         length = instr_meter_length_nbo(t, ip);
6472         color_in = (enum rte_color)ip->meter.color_in_val;
6473
6474         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6475                 &m->profile->profile,
6476                 time,
6477                 length,
6478                 color_in);
6479
6480         color_out &= m->color_mask;
6481
6482         n_pkts = m->n_pkts[color_out];
6483         n_bytes = m->n_bytes[color_out];
6484
6485         instr_meter_color_out_hbo_set(t, ip, color_out);
6486
6487         m->n_pkts[color_out] = n_pkts + 1;
6488         m->n_bytes[color_out] = n_bytes + length;
6489
6490         /* Thread. */
6491         thread_ip_inc(p);
6492 }
6493
6494 static inline void
6495 instr_meter_imm_exec(struct rte_swx_pipeline *p)
6496 {
6497         struct thread *t = &p->threads[p->thread_id];
6498         struct instruction *ip = t->ip;
6499         struct meter *m;
6500         uint64_t time, n_pkts, n_bytes;
6501         uint32_t length;
6502         enum rte_color color_in, color_out;
6503
6504         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
6505
6506         /* Structs. */
6507         m = instr_meter_idx_imm(p, ip);
6508         rte_prefetch0(m->n_pkts);
6509         time = rte_get_tsc_cycles();
6510         length = instr_meter_length_hbo(t, ip);
6511         color_in = instr_meter_color_in_hbo(t, ip);
6512
6513         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6514                 &m->profile->profile,
6515                 time,
6516                 length,
6517                 color_in);
6518
6519         color_out &= m->color_mask;
6520
6521         n_pkts = m->n_pkts[color_out];
6522         n_bytes = m->n_bytes[color_out];
6523
6524         instr_meter_color_out_hbo_set(t, ip, color_out);
6525
6526         m->n_pkts[color_out] = n_pkts + 1;
6527         m->n_bytes[color_out] = n_bytes + length;
6528
6529         /* Thread. */
6530         thread_ip_inc(p);
6531 }
6532 static inline void
6533 instr_meter_imi_exec(struct rte_swx_pipeline *p)
6534 {
6535         struct thread *t = &p->threads[p->thread_id];
6536         struct instruction *ip = t->ip;
6537         struct meter *m;
6538         uint64_t time, n_pkts, n_bytes;
6539         uint32_t length;
6540         enum rte_color color_in, color_out;
6541
6542         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
6543
6544         /* Structs. */
6545         m = instr_meter_idx_imm(p, ip);
6546         rte_prefetch0(m->n_pkts);
6547         time = rte_get_tsc_cycles();
6548         length = instr_meter_length_hbo(t, ip);
6549         color_in = (enum rte_color)ip->meter.color_in_val;
6550
6551         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6552                 &m->profile->profile,
6553                 time,
6554                 length,
6555                 color_in);
6556
6557         color_out &= m->color_mask;
6558
6559         n_pkts = m->n_pkts[color_out];
6560         n_bytes = m->n_bytes[color_out];
6561
6562         instr_meter_color_out_hbo_set(t, ip, color_out);
6563
6564         m->n_pkts[color_out] = n_pkts + 1;
6565         m->n_bytes[color_out] = n_bytes + length;
6566
6567         /* Thread. */
6568         thread_ip_inc(p);
6569 }
6570
6571 /*
6572  * jmp.
6573  */
6574 static struct action *
6575 action_find(struct rte_swx_pipeline *p, const char *name);
6576
6577 static int
6578 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
6579                     struct action *action __rte_unused,
6580                     char **tokens,
6581                     int n_tokens,
6582                     struct instruction *instr,
6583                     struct instruction_data *data)
6584 {
6585         CHECK(n_tokens == 2, EINVAL);
6586
6587         strcpy(data->jmp_label, tokens[1]);
6588
6589         instr->type = INSTR_JMP;
6590         instr->jmp.ip = NULL; /* Resolved later. */
6591         return 0;
6592 }
6593
6594 static int
6595 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
6596                           struct action *action __rte_unused,
6597                           char **tokens,
6598                           int n_tokens,
6599                           struct instruction *instr,
6600                           struct instruction_data *data)
6601 {
6602         struct header *h;
6603
6604         CHECK(n_tokens == 3, EINVAL);
6605
6606         strcpy(data->jmp_label, tokens[1]);
6607
6608         h = header_parse(p, tokens[2]);
6609         CHECK(h, EINVAL);
6610
6611         instr->type = INSTR_JMP_VALID;
6612         instr->jmp.ip = NULL; /* Resolved later. */
6613         instr->jmp.header_id = h->id;
6614         return 0;
6615 }
6616
6617 static int
6618 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
6619                             struct action *action __rte_unused,
6620                             char **tokens,
6621                             int n_tokens,
6622                             struct instruction *instr,
6623                             struct instruction_data *data)
6624 {
6625         struct header *h;
6626
6627         CHECK(n_tokens == 3, EINVAL);
6628
6629         strcpy(data->jmp_label, tokens[1]);
6630
6631         h = header_parse(p, tokens[2]);
6632         CHECK(h, EINVAL);
6633
6634         instr->type = INSTR_JMP_INVALID;
6635         instr->jmp.ip = NULL; /* Resolved later. */
6636         instr->jmp.header_id = h->id;
6637         return 0;
6638 }
6639
6640 static int
6641 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
6642                         struct action *action,
6643                         char **tokens,
6644                         int n_tokens,
6645                         struct instruction *instr,
6646                         struct instruction_data *data)
6647 {
6648         CHECK(!action, EINVAL);
6649         CHECK(n_tokens == 2, EINVAL);
6650
6651         strcpy(data->jmp_label, tokens[1]);
6652
6653         instr->type = INSTR_JMP_HIT;
6654         instr->jmp.ip = NULL; /* Resolved later. */
6655         return 0;
6656 }
6657
6658 static int
6659 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
6660                          struct action *action,
6661                          char **tokens,
6662                          int n_tokens,
6663                          struct instruction *instr,
6664                          struct instruction_data *data)
6665 {
6666         CHECK(!action, EINVAL);
6667         CHECK(n_tokens == 2, EINVAL);
6668
6669         strcpy(data->jmp_label, tokens[1]);
6670
6671         instr->type = INSTR_JMP_MISS;
6672         instr->jmp.ip = NULL; /* Resolved later. */
6673         return 0;
6674 }
6675
6676 static int
6677 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
6678                                struct action *action,
6679                                char **tokens,
6680                                int n_tokens,
6681                                struct instruction *instr,
6682                                struct instruction_data *data)
6683 {
6684         struct action *a;
6685
6686         CHECK(!action, EINVAL);
6687         CHECK(n_tokens == 3, EINVAL);
6688
6689         strcpy(data->jmp_label, tokens[1]);
6690
6691         a = action_find(p, tokens[2]);
6692         CHECK(a, EINVAL);
6693
6694         instr->type = INSTR_JMP_ACTION_HIT;
6695         instr->jmp.ip = NULL; /* Resolved later. */
6696         instr->jmp.action_id = a->id;
6697         return 0;
6698 }
6699
6700 static int
6701 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
6702                                 struct action *action,
6703                                 char **tokens,
6704                                 int n_tokens,
6705                                 struct instruction *instr,
6706                                 struct instruction_data *data)
6707 {
6708         struct action *a;
6709
6710         CHECK(!action, EINVAL);
6711         CHECK(n_tokens == 3, EINVAL);
6712
6713         strcpy(data->jmp_label, tokens[1]);
6714
6715         a = action_find(p, tokens[2]);
6716         CHECK(a, EINVAL);
6717
6718         instr->type = INSTR_JMP_ACTION_MISS;
6719         instr->jmp.ip = NULL; /* Resolved later. */
6720         instr->jmp.action_id = a->id;
6721         return 0;
6722 }
6723
6724 static int
6725 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
6726                        struct action *action,
6727                        char **tokens,
6728                        int n_tokens,
6729                        struct instruction *instr,
6730                        struct instruction_data *data)
6731 {
6732         char *a = tokens[2], *b = tokens[3];
6733         struct field *fa, *fb;
6734         uint64_t b_val;
6735         uint32_t a_struct_id, b_struct_id;
6736
6737         CHECK(n_tokens == 4, EINVAL);
6738
6739         strcpy(data->jmp_label, tokens[1]);
6740
6741         fa = struct_field_parse(p, action, a, &a_struct_id);
6742         CHECK(fa, EINVAL);
6743
6744         /* JMP_EQ or JMP_EQ_S. */
6745         fb = struct_field_parse(p, action, b, &b_struct_id);
6746         if (fb) {
6747                 instr->type = INSTR_JMP_EQ;
6748                 if ((a[0] == 'h' && b[0] != 'h') ||
6749                     (a[0] != 'h' && b[0] == 'h'))
6750                         instr->type = INSTR_JMP_EQ_S;
6751                 instr->jmp.ip = NULL; /* Resolved later. */
6752
6753                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6754                 instr->jmp.a.n_bits = fa->n_bits;
6755                 instr->jmp.a.offset = fa->offset / 8;
6756                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6757                 instr->jmp.b.n_bits = fb->n_bits;
6758                 instr->jmp.b.offset = fb->offset / 8;
6759                 return 0;
6760         }
6761
6762         /* JMP_EQ_I. */
6763         b_val = strtoull(b, &b, 0);
6764         CHECK(!b[0], EINVAL);
6765
6766         if (a[0] == 'h')
6767                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6768
6769         instr->type = INSTR_JMP_EQ_I;
6770         instr->jmp.ip = NULL; /* Resolved later. */
6771         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6772         instr->jmp.a.n_bits = fa->n_bits;
6773         instr->jmp.a.offset = fa->offset / 8;
6774         instr->jmp.b_val = b_val;
6775         return 0;
6776 }
6777
6778 static int
6779 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
6780                         struct action *action,
6781                         char **tokens,
6782                         int n_tokens,
6783                         struct instruction *instr,
6784                         struct instruction_data *data)
6785 {
6786         char *a = tokens[2], *b = tokens[3];
6787         struct field *fa, *fb;
6788         uint64_t b_val;
6789         uint32_t a_struct_id, b_struct_id;
6790
6791         CHECK(n_tokens == 4, EINVAL);
6792
6793         strcpy(data->jmp_label, tokens[1]);
6794
6795         fa = struct_field_parse(p, action, a, &a_struct_id);
6796         CHECK(fa, EINVAL);
6797
6798         /* JMP_NEQ or JMP_NEQ_S. */
6799         fb = struct_field_parse(p, action, b, &b_struct_id);
6800         if (fb) {
6801                 instr->type = INSTR_JMP_NEQ;
6802                 if ((a[0] == 'h' && b[0] != 'h') ||
6803                     (a[0] != 'h' && b[0] == 'h'))
6804                         instr->type = INSTR_JMP_NEQ_S;
6805                 instr->jmp.ip = NULL; /* Resolved later. */
6806
6807                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6808                 instr->jmp.a.n_bits = fa->n_bits;
6809                 instr->jmp.a.offset = fa->offset / 8;
6810                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6811                 instr->jmp.b.n_bits = fb->n_bits;
6812                 instr->jmp.b.offset = fb->offset / 8;
6813                 return 0;
6814         }
6815
6816         /* JMP_NEQ_I. */
6817         b_val = strtoull(b, &b, 0);
6818         CHECK(!b[0], EINVAL);
6819
6820         if (a[0] == 'h')
6821                 b_val = hton64(b_val) >> (64 - fa->n_bits);
6822
6823         instr->type = INSTR_JMP_NEQ_I;
6824         instr->jmp.ip = NULL; /* Resolved later. */
6825         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6826         instr->jmp.a.n_bits = fa->n_bits;
6827         instr->jmp.a.offset = fa->offset / 8;
6828         instr->jmp.b_val = b_val;
6829         return 0;
6830 }
6831
6832 static int
6833 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
6834                        struct action *action,
6835                        char **tokens,
6836                        int n_tokens,
6837                        struct instruction *instr,
6838                        struct instruction_data *data)
6839 {
6840         char *a = tokens[2], *b = tokens[3];
6841         struct field *fa, *fb;
6842         uint64_t b_val;
6843         uint32_t a_struct_id, b_struct_id;
6844
6845         CHECK(n_tokens == 4, EINVAL);
6846
6847         strcpy(data->jmp_label, tokens[1]);
6848
6849         fa = struct_field_parse(p, action, a, &a_struct_id);
6850         CHECK(fa, EINVAL);
6851
6852         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
6853         fb = struct_field_parse(p, action, b, &b_struct_id);
6854         if (fb) {
6855                 instr->type = INSTR_JMP_LT;
6856                 if (a[0] == 'h' && b[0] != 'h')
6857                         instr->type = INSTR_JMP_LT_HM;
6858                 if (a[0] != 'h' && b[0] == 'h')
6859                         instr->type = INSTR_JMP_LT_MH;
6860                 if (a[0] == 'h' && b[0] == 'h')
6861                         instr->type = INSTR_JMP_LT_HH;
6862                 instr->jmp.ip = NULL; /* Resolved later. */
6863
6864                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6865                 instr->jmp.a.n_bits = fa->n_bits;
6866                 instr->jmp.a.offset = fa->offset / 8;
6867                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6868                 instr->jmp.b.n_bits = fb->n_bits;
6869                 instr->jmp.b.offset = fb->offset / 8;
6870                 return 0;
6871         }
6872
6873         /* JMP_LT_MI, JMP_LT_HI. */
6874         b_val = strtoull(b, &b, 0);
6875         CHECK(!b[0], EINVAL);
6876
6877         instr->type = INSTR_JMP_LT_MI;
6878         if (a[0] == 'h')
6879                 instr->type = INSTR_JMP_LT_HI;
6880         instr->jmp.ip = NULL; /* Resolved later. */
6881
6882         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6883         instr->jmp.a.n_bits = fa->n_bits;
6884         instr->jmp.a.offset = fa->offset / 8;
6885         instr->jmp.b_val = b_val;
6886         return 0;
6887 }
6888
6889 static int
6890 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
6891                        struct action *action,
6892                        char **tokens,
6893                        int n_tokens,
6894                        struct instruction *instr,
6895                        struct instruction_data *data)
6896 {
6897         char *a = tokens[2], *b = tokens[3];
6898         struct field *fa, *fb;
6899         uint64_t b_val;
6900         uint32_t a_struct_id, b_struct_id;
6901
6902         CHECK(n_tokens == 4, EINVAL);
6903
6904         strcpy(data->jmp_label, tokens[1]);
6905
6906         fa = struct_field_parse(p, action, a, &a_struct_id);
6907         CHECK(fa, EINVAL);
6908
6909         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
6910         fb = struct_field_parse(p, action, b, &b_struct_id);
6911         if (fb) {
6912                 instr->type = INSTR_JMP_GT;
6913                 if (a[0] == 'h' && b[0] != 'h')
6914                         instr->type = INSTR_JMP_GT_HM;
6915                 if (a[0] != 'h' && b[0] == 'h')
6916                         instr->type = INSTR_JMP_GT_MH;
6917                 if (a[0] == 'h' && b[0] == 'h')
6918                         instr->type = INSTR_JMP_GT_HH;
6919                 instr->jmp.ip = NULL; /* Resolved later. */
6920
6921                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6922                 instr->jmp.a.n_bits = fa->n_bits;
6923                 instr->jmp.a.offset = fa->offset / 8;
6924                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6925                 instr->jmp.b.n_bits = fb->n_bits;
6926                 instr->jmp.b.offset = fb->offset / 8;
6927                 return 0;
6928         }
6929
6930         /* JMP_GT_MI, JMP_GT_HI. */
6931         b_val = strtoull(b, &b, 0);
6932         CHECK(!b[0], EINVAL);
6933
6934         instr->type = INSTR_JMP_GT_MI;
6935         if (a[0] == 'h')
6936                 instr->type = INSTR_JMP_GT_HI;
6937         instr->jmp.ip = NULL; /* Resolved later. */
6938
6939         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6940         instr->jmp.a.n_bits = fa->n_bits;
6941         instr->jmp.a.offset = fa->offset / 8;
6942         instr->jmp.b_val = b_val;
6943         return 0;
6944 }
6945
6946 static inline void
6947 instr_jmp_exec(struct rte_swx_pipeline *p)
6948 {
6949         struct thread *t = &p->threads[p->thread_id];
6950         struct instruction *ip = t->ip;
6951
6952         TRACE("[Thread %2u] jmp\n", p->thread_id);
6953
6954         thread_ip_set(t, ip->jmp.ip);
6955 }
6956
6957 static inline void
6958 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
6959 {
6960         struct thread *t = &p->threads[p->thread_id];
6961         struct instruction *ip = t->ip;
6962         uint32_t header_id = ip->jmp.header_id;
6963
6964         TRACE("[Thread %2u] jmpv\n", p->thread_id);
6965
6966         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
6967 }
6968
6969 static inline void
6970 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
6971 {
6972         struct thread *t = &p->threads[p->thread_id];
6973         struct instruction *ip = t->ip;
6974         uint32_t header_id = ip->jmp.header_id;
6975
6976         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
6977
6978         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
6979 }
6980
6981 static inline void
6982 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
6983 {
6984         struct thread *t = &p->threads[p->thread_id];
6985         struct instruction *ip = t->ip;
6986         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
6987
6988         TRACE("[Thread %2u] jmph\n", p->thread_id);
6989
6990         t->ip = ip_next[t->hit];
6991 }
6992
6993 static inline void
6994 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
6995 {
6996         struct thread *t = &p->threads[p->thread_id];
6997         struct instruction *ip = t->ip;
6998         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
6999
7000         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
7001
7002         t->ip = ip_next[t->hit];
7003 }
7004
7005 static inline void
7006 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
7007 {
7008         struct thread *t = &p->threads[p->thread_id];
7009         struct instruction *ip = t->ip;
7010
7011         TRACE("[Thread %2u] jmpa\n", p->thread_id);
7012
7013         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
7014 }
7015
7016 static inline void
7017 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
7018 {
7019         struct thread *t = &p->threads[p->thread_id];
7020         struct instruction *ip = t->ip;
7021
7022         TRACE("[Thread %2u] jmpna\n", p->thread_id);
7023
7024         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
7025 }
7026
7027 static inline void
7028 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
7029 {
7030         struct thread *t = &p->threads[p->thread_id];
7031         struct instruction *ip = t->ip;
7032
7033         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
7034
7035         JMP_CMP(t, ip, ==);
7036 }
7037
7038 static inline void
7039 instr_jmp_eq_s_exec(struct rte_swx_pipeline *p)
7040 {
7041         struct thread *t = &p->threads[p->thread_id];
7042         struct instruction *ip = t->ip;
7043
7044         TRACE("[Thread %2u] jmpeq (s)\n", p->thread_id);
7045
7046         JMP_CMP_S(t, ip, ==);
7047 }
7048
7049 static inline void
7050 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
7051 {
7052         struct thread *t = &p->threads[p->thread_id];
7053         struct instruction *ip = t->ip;
7054
7055         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
7056
7057         JMP_CMP_I(t, ip, ==);
7058 }
7059
7060 static inline void
7061 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
7062 {
7063         struct thread *t = &p->threads[p->thread_id];
7064         struct instruction *ip = t->ip;
7065
7066         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
7067
7068         JMP_CMP(t, ip, !=);
7069 }
7070
7071 static inline void
7072 instr_jmp_neq_s_exec(struct rte_swx_pipeline *p)
7073 {
7074         struct thread *t = &p->threads[p->thread_id];
7075         struct instruction *ip = t->ip;
7076
7077         TRACE("[Thread %2u] jmpneq (s)\n", p->thread_id);
7078
7079         JMP_CMP_S(t, ip, !=);
7080 }
7081
7082 static inline void
7083 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
7084 {
7085         struct thread *t = &p->threads[p->thread_id];
7086         struct instruction *ip = t->ip;
7087
7088         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
7089
7090         JMP_CMP_I(t, ip, !=);
7091 }
7092
7093 static inline void
7094 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
7095 {
7096         struct thread *t = &p->threads[p->thread_id];
7097         struct instruction *ip = t->ip;
7098
7099         TRACE("[Thread %2u] jmplt\n", p->thread_id);
7100
7101         JMP_CMP(t, ip, <);
7102 }
7103
7104 static inline void
7105 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
7106 {
7107         struct thread *t = &p->threads[p->thread_id];
7108         struct instruction *ip = t->ip;
7109
7110         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
7111
7112         JMP_CMP_MH(t, ip, <);
7113 }
7114
7115 static inline void
7116 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
7117 {
7118         struct thread *t = &p->threads[p->thread_id];
7119         struct instruction *ip = t->ip;
7120
7121         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
7122
7123         JMP_CMP_HM(t, ip, <);
7124 }
7125
7126 static inline void
7127 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
7128 {
7129         struct thread *t = &p->threads[p->thread_id];
7130         struct instruction *ip = t->ip;
7131
7132         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
7133
7134         JMP_CMP_HH(t, ip, <);
7135 }
7136
7137 static inline void
7138 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
7139 {
7140         struct thread *t = &p->threads[p->thread_id];
7141         struct instruction *ip = t->ip;
7142
7143         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
7144
7145         JMP_CMP_MI(t, ip, <);
7146 }
7147
7148 static inline void
7149 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
7150 {
7151         struct thread *t = &p->threads[p->thread_id];
7152         struct instruction *ip = t->ip;
7153
7154         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
7155
7156         JMP_CMP_HI(t, ip, <);
7157 }
7158
7159 static inline void
7160 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
7161 {
7162         struct thread *t = &p->threads[p->thread_id];
7163         struct instruction *ip = t->ip;
7164
7165         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
7166
7167         JMP_CMP(t, ip, >);
7168 }
7169
7170 static inline void
7171 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
7172 {
7173         struct thread *t = &p->threads[p->thread_id];
7174         struct instruction *ip = t->ip;
7175
7176         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
7177
7178         JMP_CMP_MH(t, ip, >);
7179 }
7180
7181 static inline void
7182 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
7183 {
7184         struct thread *t = &p->threads[p->thread_id];
7185         struct instruction *ip = t->ip;
7186
7187         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
7188
7189         JMP_CMP_HM(t, ip, >);
7190 }
7191
7192 static inline void
7193 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
7194 {
7195         struct thread *t = &p->threads[p->thread_id];
7196         struct instruction *ip = t->ip;
7197
7198         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
7199
7200         JMP_CMP_HH(t, ip, >);
7201 }
7202
7203 static inline void
7204 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
7205 {
7206         struct thread *t = &p->threads[p->thread_id];
7207         struct instruction *ip = t->ip;
7208
7209         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
7210
7211         JMP_CMP_MI(t, ip, >);
7212 }
7213
7214 static inline void
7215 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
7216 {
7217         struct thread *t = &p->threads[p->thread_id];
7218         struct instruction *ip = t->ip;
7219
7220         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
7221
7222         JMP_CMP_HI(t, ip, >);
7223 }
7224
7225 /*
7226  * return.
7227  */
7228 static int
7229 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
7230                        struct action *action,
7231                        char **tokens __rte_unused,
7232                        int n_tokens,
7233                        struct instruction *instr,
7234                        struct instruction_data *data __rte_unused)
7235 {
7236         CHECK(action, EINVAL);
7237         CHECK(n_tokens == 1, EINVAL);
7238
7239         instr->type = INSTR_RETURN;
7240         return 0;
7241 }
7242
7243 static inline void
7244 instr_return_exec(struct rte_swx_pipeline *p)
7245 {
7246         struct thread *t = &p->threads[p->thread_id];
7247
7248         TRACE("[Thread %2u] return\n", p->thread_id);
7249
7250         t->ip = t->ret;
7251 }
7252
7253 static int
7254 instr_translate(struct rte_swx_pipeline *p,
7255                 struct action *action,
7256                 char *string,
7257                 struct instruction *instr,
7258                 struct instruction_data *data)
7259 {
7260         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
7261         int n_tokens = 0, tpos = 0;
7262
7263         /* Parse the instruction string into tokens. */
7264         for ( ; ; ) {
7265                 char *token;
7266
7267                 token = strtok_r(string, " \t\v", &string);
7268                 if (!token)
7269                         break;
7270
7271                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
7272                 CHECK_NAME(token, EINVAL);
7273
7274                 tokens[n_tokens] = token;
7275                 n_tokens++;
7276         }
7277
7278         CHECK(n_tokens, EINVAL);
7279
7280         /* Handle the optional instruction label. */
7281         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
7282                 strcpy(data->label, tokens[0]);
7283
7284                 tpos += 2;
7285                 CHECK(n_tokens - tpos, EINVAL);
7286         }
7287
7288         /* Identify the instruction type. */
7289         if (!strcmp(tokens[tpos], "rx"))
7290                 return instr_rx_translate(p,
7291                                           action,
7292                                           &tokens[tpos],
7293                                           n_tokens - tpos,
7294                                           instr,
7295                                           data);
7296
7297         if (!strcmp(tokens[tpos], "tx"))
7298                 return instr_tx_translate(p,
7299                                           action,
7300                                           &tokens[tpos],
7301                                           n_tokens - tpos,
7302                                           instr,
7303                                           data);
7304
7305         if (!strcmp(tokens[tpos], "extract"))
7306                 return instr_hdr_extract_translate(p,
7307                                                    action,
7308                                                    &tokens[tpos],
7309                                                    n_tokens - tpos,
7310                                                    instr,
7311                                                    data);
7312
7313         if (!strcmp(tokens[tpos], "emit"))
7314                 return instr_hdr_emit_translate(p,
7315                                                 action,
7316                                                 &tokens[tpos],
7317                                                 n_tokens - tpos,
7318                                                 instr,
7319                                                 data);
7320
7321         if (!strcmp(tokens[tpos], "validate"))
7322                 return instr_hdr_validate_translate(p,
7323                                                     action,
7324                                                     &tokens[tpos],
7325                                                     n_tokens - tpos,
7326                                                     instr,
7327                                                     data);
7328
7329         if (!strcmp(tokens[tpos], "invalidate"))
7330                 return instr_hdr_invalidate_translate(p,
7331                                                       action,
7332                                                       &tokens[tpos],
7333                                                       n_tokens - tpos,
7334                                                       instr,
7335                                                       data);
7336
7337         if (!strcmp(tokens[tpos], "mov"))
7338                 return instr_mov_translate(p,
7339                                            action,
7340                                            &tokens[tpos],
7341                                            n_tokens - tpos,
7342                                            instr,
7343                                            data);
7344
7345         if (!strcmp(tokens[tpos], "dma"))
7346                 return instr_dma_translate(p,
7347                                            action,
7348                                            &tokens[tpos],
7349                                            n_tokens - tpos,
7350                                            instr,
7351                                            data);
7352
7353         if (!strcmp(tokens[tpos], "add"))
7354                 return instr_alu_add_translate(p,
7355                                                action,
7356                                                &tokens[tpos],
7357                                                n_tokens - tpos,
7358                                                instr,
7359                                                data);
7360
7361         if (!strcmp(tokens[tpos], "sub"))
7362                 return instr_alu_sub_translate(p,
7363                                                action,
7364                                                &tokens[tpos],
7365                                                n_tokens - tpos,
7366                                                instr,
7367                                                data);
7368
7369         if (!strcmp(tokens[tpos], "ckadd"))
7370                 return instr_alu_ckadd_translate(p,
7371                                                  action,
7372                                                  &tokens[tpos],
7373                                                  n_tokens - tpos,
7374                                                  instr,
7375                                                  data);
7376
7377         if (!strcmp(tokens[tpos], "cksub"))
7378                 return instr_alu_cksub_translate(p,
7379                                                  action,
7380                                                  &tokens[tpos],
7381                                                  n_tokens - tpos,
7382                                                  instr,
7383                                                  data);
7384
7385         if (!strcmp(tokens[tpos], "and"))
7386                 return instr_alu_and_translate(p,
7387                                                action,
7388                                                &tokens[tpos],
7389                                                n_tokens - tpos,
7390                                                instr,
7391                                                data);
7392
7393         if (!strcmp(tokens[tpos], "or"))
7394                 return instr_alu_or_translate(p,
7395                                               action,
7396                                               &tokens[tpos],
7397                                               n_tokens - tpos,
7398                                               instr,
7399                                               data);
7400
7401         if (!strcmp(tokens[tpos], "xor"))
7402                 return instr_alu_xor_translate(p,
7403                                                action,
7404                                                &tokens[tpos],
7405                                                n_tokens - tpos,
7406                                                instr,
7407                                                data);
7408
7409         if (!strcmp(tokens[tpos], "shl"))
7410                 return instr_alu_shl_translate(p,
7411                                                action,
7412                                                &tokens[tpos],
7413                                                n_tokens - tpos,
7414                                                instr,
7415                                                data);
7416
7417         if (!strcmp(tokens[tpos], "shr"))
7418                 return instr_alu_shr_translate(p,
7419                                                action,
7420                                                &tokens[tpos],
7421                                                n_tokens - tpos,
7422                                                instr,
7423                                                data);
7424
7425         if (!strcmp(tokens[tpos], "regprefetch"))
7426                 return instr_regprefetch_translate(p,
7427                                                    action,
7428                                                    &tokens[tpos],
7429                                                    n_tokens - tpos,
7430                                                    instr,
7431                                                    data);
7432
7433         if (!strcmp(tokens[tpos], "regrd"))
7434                 return instr_regrd_translate(p,
7435                                              action,
7436                                              &tokens[tpos],
7437                                              n_tokens - tpos,
7438                                              instr,
7439                                              data);
7440
7441         if (!strcmp(tokens[tpos], "regwr"))
7442                 return instr_regwr_translate(p,
7443                                              action,
7444                                              &tokens[tpos],
7445                                              n_tokens - tpos,
7446                                              instr,
7447                                              data);
7448
7449         if (!strcmp(tokens[tpos], "regadd"))
7450                 return instr_regadd_translate(p,
7451                                               action,
7452                                               &tokens[tpos],
7453                                               n_tokens - tpos,
7454                                               instr,
7455                                               data);
7456
7457         if (!strcmp(tokens[tpos], "metprefetch"))
7458                 return instr_metprefetch_translate(p,
7459                                                    action,
7460                                                    &tokens[tpos],
7461                                                    n_tokens - tpos,
7462                                                    instr,
7463                                                    data);
7464
7465         if (!strcmp(tokens[tpos], "meter"))
7466                 return instr_meter_translate(p,
7467                                              action,
7468                                              &tokens[tpos],
7469                                              n_tokens - tpos,
7470                                              instr,
7471                                              data);
7472
7473         if (!strcmp(tokens[tpos], "table"))
7474                 return instr_table_translate(p,
7475                                              action,
7476                                              &tokens[tpos],
7477                                              n_tokens - tpos,
7478                                              instr,
7479                                              data);
7480
7481         if (!strcmp(tokens[tpos], "extern"))
7482                 return instr_extern_translate(p,
7483                                               action,
7484                                               &tokens[tpos],
7485                                               n_tokens - tpos,
7486                                               instr,
7487                                               data);
7488
7489         if (!strcmp(tokens[tpos], "jmp"))
7490                 return instr_jmp_translate(p,
7491                                            action,
7492                                            &tokens[tpos],
7493                                            n_tokens - tpos,
7494                                            instr,
7495                                            data);
7496
7497         if (!strcmp(tokens[tpos], "jmpv"))
7498                 return instr_jmp_valid_translate(p,
7499                                                  action,
7500                                                  &tokens[tpos],
7501                                                  n_tokens - tpos,
7502                                                  instr,
7503                                                  data);
7504
7505         if (!strcmp(tokens[tpos], "jmpnv"))
7506                 return instr_jmp_invalid_translate(p,
7507                                                    action,
7508                                                    &tokens[tpos],
7509                                                    n_tokens - tpos,
7510                                                    instr,
7511                                                    data);
7512
7513         if (!strcmp(tokens[tpos], "jmph"))
7514                 return instr_jmp_hit_translate(p,
7515                                                action,
7516                                                &tokens[tpos],
7517                                                n_tokens - tpos,
7518                                                instr,
7519                                                data);
7520
7521         if (!strcmp(tokens[tpos], "jmpnh"))
7522                 return instr_jmp_miss_translate(p,
7523                                                 action,
7524                                                 &tokens[tpos],
7525                                                 n_tokens - tpos,
7526                                                 instr,
7527                                                 data);
7528
7529         if (!strcmp(tokens[tpos], "jmpa"))
7530                 return instr_jmp_action_hit_translate(p,
7531                                                       action,
7532                                                       &tokens[tpos],
7533                                                       n_tokens - tpos,
7534                                                       instr,
7535                                                       data);
7536
7537         if (!strcmp(tokens[tpos], "jmpna"))
7538                 return instr_jmp_action_miss_translate(p,
7539                                                        action,
7540                                                        &tokens[tpos],
7541                                                        n_tokens - tpos,
7542                                                        instr,
7543                                                        data);
7544
7545         if (!strcmp(tokens[tpos], "jmpeq"))
7546                 return instr_jmp_eq_translate(p,
7547                                               action,
7548                                               &tokens[tpos],
7549                                               n_tokens - tpos,
7550                                               instr,
7551                                               data);
7552
7553         if (!strcmp(tokens[tpos], "jmpneq"))
7554                 return instr_jmp_neq_translate(p,
7555                                                action,
7556                                                &tokens[tpos],
7557                                                n_tokens - tpos,
7558                                                instr,
7559                                                data);
7560
7561         if (!strcmp(tokens[tpos], "jmplt"))
7562                 return instr_jmp_lt_translate(p,
7563                                               action,
7564                                               &tokens[tpos],
7565                                               n_tokens - tpos,
7566                                               instr,
7567                                               data);
7568
7569         if (!strcmp(tokens[tpos], "jmpgt"))
7570                 return instr_jmp_gt_translate(p,
7571                                               action,
7572                                               &tokens[tpos],
7573                                               n_tokens - tpos,
7574                                               instr,
7575                                               data);
7576
7577         if (!strcmp(tokens[tpos], "return"))
7578                 return instr_return_translate(p,
7579                                               action,
7580                                               &tokens[tpos],
7581                                               n_tokens - tpos,
7582                                               instr,
7583                                               data);
7584
7585         CHECK(0, EINVAL);
7586 }
7587
7588 static struct instruction_data *
7589 label_find(struct instruction_data *data, uint32_t n, const char *label)
7590 {
7591         uint32_t i;
7592
7593         for (i = 0; i < n; i++)
7594                 if (!strcmp(label, data[i].label))
7595                         return &data[i];
7596
7597         return NULL;
7598 }
7599
7600 static uint32_t
7601 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
7602 {
7603         uint32_t count = 0, i;
7604
7605         if (!label[0])
7606                 return 0;
7607
7608         for (i = 0; i < n; i++)
7609                 if (!strcmp(label, data[i].jmp_label))
7610                         count++;
7611
7612         return count;
7613 }
7614
7615 static int
7616 instr_label_check(struct instruction_data *instruction_data,
7617                   uint32_t n_instructions)
7618 {
7619         uint32_t i;
7620
7621         /* Check that all instruction labels are unique. */
7622         for (i = 0; i < n_instructions; i++) {
7623                 struct instruction_data *data = &instruction_data[i];
7624                 char *label = data->label;
7625                 uint32_t j;
7626
7627                 if (!label[0])
7628                         continue;
7629
7630                 for (j = i + 1; j < n_instructions; j++)
7631                         CHECK(strcmp(label, data[j].label), EINVAL);
7632         }
7633
7634         /* Get users for each instruction label. */
7635         for (i = 0; i < n_instructions; i++) {
7636                 struct instruction_data *data = &instruction_data[i];
7637                 char *label = data->label;
7638
7639                 data->n_users = label_is_used(instruction_data,
7640                                               n_instructions,
7641                                               label);
7642         }
7643
7644         return 0;
7645 }
7646
7647 static int
7648 instr_jmp_resolve(struct instruction *instructions,
7649                   struct instruction_data *instruction_data,
7650                   uint32_t n_instructions)
7651 {
7652         uint32_t i;
7653
7654         for (i = 0; i < n_instructions; i++) {
7655                 struct instruction *instr = &instructions[i];
7656                 struct instruction_data *data = &instruction_data[i];
7657                 struct instruction_data *found;
7658
7659                 if (!instruction_is_jmp(instr))
7660                         continue;
7661
7662                 found = label_find(instruction_data,
7663                                    n_instructions,
7664                                    data->jmp_label);
7665                 CHECK(found, EINVAL);
7666
7667                 instr->jmp.ip = &instructions[found - instruction_data];
7668         }
7669
7670         return 0;
7671 }
7672
7673 static int
7674 instr_verify(struct rte_swx_pipeline *p __rte_unused,
7675              struct action *a,
7676              struct instruction *instr,
7677              struct instruction_data *data __rte_unused,
7678              uint32_t n_instructions)
7679 {
7680         if (!a) {
7681                 enum instruction_type type;
7682                 uint32_t i;
7683
7684                 /* Check that the first instruction is rx. */
7685                 CHECK(instr[0].type == INSTR_RX, EINVAL);
7686
7687                 /* Check that there is at least one tx instruction. */
7688                 for (i = 0; i < n_instructions; i++) {
7689                         type = instr[i].type;
7690
7691                         if (type == INSTR_TX)
7692                                 break;
7693                 }
7694                 CHECK(i < n_instructions, EINVAL);
7695
7696                 /* Check that the last instruction is either tx or unconditional
7697                  * jump.
7698                  */
7699                 type = instr[n_instructions - 1].type;
7700                 CHECK((type == INSTR_TX) || (type == INSTR_JMP), EINVAL);
7701         }
7702
7703         if (a) {
7704                 enum instruction_type type;
7705                 uint32_t i;
7706
7707                 /* Check that there is at least one return or tx instruction. */
7708                 for (i = 0; i < n_instructions; i++) {
7709                         type = instr[i].type;
7710
7711                         if ((type == INSTR_RETURN) || (type == INSTR_TX))
7712                                 break;
7713                 }
7714                 CHECK(i < n_instructions, EINVAL);
7715         }
7716
7717         return 0;
7718 }
7719
7720 static int
7721 instr_pattern_extract_many_detect(struct instruction *instr,
7722                                   struct instruction_data *data,
7723                                   uint32_t n_instr,
7724                                   uint32_t *n_pattern_instr)
7725 {
7726         uint32_t i;
7727
7728         for (i = 0; i < n_instr; i++) {
7729                 if (data[i].invalid)
7730                         break;
7731
7732                 if (instr[i].type != INSTR_HDR_EXTRACT)
7733                         break;
7734
7735                 if (i == RTE_DIM(instr->io.hdr.header_id))
7736                         break;
7737
7738                 if (i && data[i].n_users)
7739                         break;
7740         }
7741
7742         if (i < 2)
7743                 return 0;
7744
7745         *n_pattern_instr = i;
7746         return 1;
7747 }
7748
7749 static void
7750 instr_pattern_extract_many_optimize(struct instruction *instr,
7751                                     struct instruction_data *data,
7752                                     uint32_t n_instr)
7753 {
7754         uint32_t i;
7755
7756         for (i = 1; i < n_instr; i++) {
7757                 instr[0].type++;
7758                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7759                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7760                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7761
7762                 data[i].invalid = 1;
7763         }
7764 }
7765
7766 static int
7767 instr_pattern_emit_many_tx_detect(struct instruction *instr,
7768                                   struct instruction_data *data,
7769                                   uint32_t n_instr,
7770                                   uint32_t *n_pattern_instr)
7771 {
7772         uint32_t i;
7773
7774         for (i = 0; i < n_instr; i++) {
7775                 if (data[i].invalid)
7776                         break;
7777
7778                 if (instr[i].type != INSTR_HDR_EMIT)
7779                         break;
7780
7781                 if (i == RTE_DIM(instr->io.hdr.header_id))
7782                         break;
7783
7784                 if (i && data[i].n_users)
7785                         break;
7786         }
7787
7788         if (!i)
7789                 return 0;
7790
7791         if (instr[i].type != INSTR_TX)
7792                 return 0;
7793
7794         if (data[i].n_users)
7795                 return 0;
7796
7797         i++;
7798
7799         *n_pattern_instr = i;
7800         return 1;
7801 }
7802
7803 static void
7804 instr_pattern_emit_many_tx_optimize(struct instruction *instr,
7805                                     struct instruction_data *data,
7806                                     uint32_t n_instr)
7807 {
7808         uint32_t i;
7809
7810         /* Any emit instruction in addition to the first one. */
7811         for (i = 1; i < n_instr - 1; i++) {
7812                 instr[0].type++;
7813                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7814                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7815                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7816
7817                 data[i].invalid = 1;
7818         }
7819
7820         /* The TX instruction is the last one in the pattern. */
7821         instr[0].type++;
7822         instr[0].io.io.offset = instr[i].io.io.offset;
7823         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7824         data[i].invalid = 1;
7825 }
7826
7827 static int
7828 instr_pattern_dma_many_detect(struct instruction *instr,
7829                               struct instruction_data *data,
7830                               uint32_t n_instr,
7831                               uint32_t *n_pattern_instr)
7832 {
7833         uint32_t i;
7834
7835         for (i = 0; i < n_instr; i++) {
7836                 if (data[i].invalid)
7837                         break;
7838
7839                 if (instr[i].type != INSTR_DMA_HT)
7840                         break;
7841
7842                 if (i == RTE_DIM(instr->dma.dst.header_id))
7843                         break;
7844
7845                 if (i && data[i].n_users)
7846                         break;
7847         }
7848
7849         if (i < 2)
7850                 return 0;
7851
7852         *n_pattern_instr = i;
7853         return 1;
7854 }
7855
7856 static void
7857 instr_pattern_dma_many_optimize(struct instruction *instr,
7858                                 struct instruction_data *data,
7859                                 uint32_t n_instr)
7860 {
7861         uint32_t i;
7862
7863         for (i = 1; i < n_instr; i++) {
7864                 instr[0].type++;
7865                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7866                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7867                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7868                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7869
7870                 data[i].invalid = 1;
7871         }
7872 }
7873
7874 static uint32_t
7875 instr_optimize(struct instruction *instructions,
7876                struct instruction_data *instruction_data,
7877                uint32_t n_instructions)
7878 {
7879         uint32_t i, pos = 0;
7880
7881         for (i = 0; i < n_instructions; ) {
7882                 struct instruction *instr = &instructions[i];
7883                 struct instruction_data *data = &instruction_data[i];
7884                 uint32_t n_instr = 0;
7885                 int detected;
7886
7887                 /* Extract many. */
7888                 detected = instr_pattern_extract_many_detect(instr,
7889                                                              data,
7890                                                              n_instructions - i,
7891                                                              &n_instr);
7892                 if (detected) {
7893                         instr_pattern_extract_many_optimize(instr,
7894                                                             data,
7895                                                             n_instr);
7896                         i += n_instr;
7897                         continue;
7898                 }
7899
7900                 /* Emit many + TX. */
7901                 detected = instr_pattern_emit_many_tx_detect(instr,
7902                                                              data,
7903                                                              n_instructions - i,
7904                                                              &n_instr);
7905                 if (detected) {
7906                         instr_pattern_emit_many_tx_optimize(instr,
7907                                                             data,
7908                                                             n_instr);
7909                         i += n_instr;
7910                         continue;
7911                 }
7912
7913                 /* DMA many. */
7914                 detected = instr_pattern_dma_many_detect(instr,
7915                                                          data,
7916                                                          n_instructions - i,
7917                                                          &n_instr);
7918                 if (detected) {
7919                         instr_pattern_dma_many_optimize(instr, data, n_instr);
7920                         i += n_instr;
7921                         continue;
7922                 }
7923
7924                 /* No pattern starting at the current instruction. */
7925                 i++;
7926         }
7927
7928         /* Eliminate the invalid instructions that have been optimized out. */
7929         for (i = 0; i < n_instructions; i++) {
7930                 struct instruction *instr = &instructions[i];
7931                 struct instruction_data *data = &instruction_data[i];
7932
7933                 if (data->invalid)
7934                         continue;
7935
7936                 if (i != pos) {
7937                         memcpy(&instructions[pos], instr, sizeof(*instr));
7938                         memcpy(&instruction_data[pos], data, sizeof(*data));
7939                 }
7940
7941                 pos++;
7942         }
7943
7944         return pos;
7945 }
7946
7947 static int
7948 instruction_config(struct rte_swx_pipeline *p,
7949                    struct action *a,
7950                    const char **instructions,
7951                    uint32_t n_instructions)
7952 {
7953         struct instruction *instr = NULL;
7954         struct instruction_data *data = NULL;
7955         int err = 0;
7956         uint32_t i;
7957
7958         CHECK(n_instructions, EINVAL);
7959         CHECK(instructions, EINVAL);
7960         for (i = 0; i < n_instructions; i++)
7961                 CHECK_INSTRUCTION(instructions[i], EINVAL);
7962
7963         /* Memory allocation. */
7964         instr = calloc(n_instructions, sizeof(struct instruction));
7965         if (!instr) {
7966                 err = ENOMEM;
7967                 goto error;
7968         }
7969
7970         data = calloc(n_instructions, sizeof(struct instruction_data));
7971         if (!data) {
7972                 err = ENOMEM;
7973                 goto error;
7974         }
7975
7976         for (i = 0; i < n_instructions; i++) {
7977                 char *string = strdup(instructions[i]);
7978                 if (!string) {
7979                         err = ENOMEM;
7980                         goto error;
7981                 }
7982
7983                 err = instr_translate(p, a, string, &instr[i], &data[i]);
7984                 if (err) {
7985                         free(string);
7986                         goto error;
7987                 }
7988
7989                 free(string);
7990         }
7991
7992         err = instr_label_check(data, n_instructions);
7993         if (err)
7994                 goto error;
7995
7996         err = instr_verify(p, a, instr, data, n_instructions);
7997         if (err)
7998                 goto error;
7999
8000         n_instructions = instr_optimize(instr, data, n_instructions);
8001
8002         err = instr_jmp_resolve(instr, data, n_instructions);
8003         if (err)
8004                 goto error;
8005
8006         if (a) {
8007                 a->instructions = instr;
8008                 a->n_instructions = n_instructions;
8009         } else {
8010                 p->instructions = instr;
8011                 p->n_instructions = n_instructions;
8012         }
8013
8014         free(data);
8015         return 0;
8016
8017 error:
8018         free(data);
8019         free(instr);
8020         return err;
8021 }
8022
8023 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
8024
8025 static instr_exec_t instruction_table[] = {
8026         [INSTR_RX] = instr_rx_exec,
8027         [INSTR_TX] = instr_tx_exec,
8028
8029         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
8030         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
8031         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
8032         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
8033         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
8034         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
8035         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
8036         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
8037
8038         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
8039         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
8040         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
8041         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
8042         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
8043         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
8044         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
8045         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
8046         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
8047
8048         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
8049         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
8050
8051         [INSTR_MOV] = instr_mov_exec,
8052         [INSTR_MOV_S] = instr_mov_s_exec,
8053         [INSTR_MOV_I] = instr_mov_i_exec,
8054
8055         [INSTR_DMA_HT] = instr_dma_ht_exec,
8056         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
8057         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
8058         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
8059         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
8060         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
8061         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
8062         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
8063
8064         [INSTR_ALU_ADD] = instr_alu_add_exec,
8065         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
8066         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
8067         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
8068         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
8069         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
8070
8071         [INSTR_ALU_SUB] = instr_alu_sub_exec,
8072         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
8073         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
8074         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
8075         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
8076         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
8077
8078         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
8079         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
8080         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
8081         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
8082
8083         [INSTR_ALU_AND] = instr_alu_and_exec,
8084         [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
8085         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
8086
8087         [INSTR_ALU_OR] = instr_alu_or_exec,
8088         [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
8089         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
8090
8091         [INSTR_ALU_XOR] = instr_alu_xor_exec,
8092         [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec,
8093         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
8094
8095         [INSTR_ALU_SHL] = instr_alu_shl_exec,
8096         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
8097         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
8098         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
8099         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
8100         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
8101
8102         [INSTR_ALU_SHR] = instr_alu_shr_exec,
8103         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
8104         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
8105         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
8106         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
8107         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
8108
8109         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
8110         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
8111         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
8112
8113         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
8114         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
8115         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
8116         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
8117         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
8118         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
8119
8120         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
8121         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
8122         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
8123         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
8124         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
8125         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
8126         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
8127         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
8128         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
8129
8130         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
8131         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
8132         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
8133         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
8134         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
8135         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
8136         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
8137         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
8138         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
8139
8140         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
8141         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
8142         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
8143
8144         [INSTR_METER_HHM] = instr_meter_hhm_exec,
8145         [INSTR_METER_HHI] = instr_meter_hhi_exec,
8146         [INSTR_METER_HMM] = instr_meter_hmm_exec,
8147         [INSTR_METER_HMI] = instr_meter_hmi_exec,
8148         [INSTR_METER_MHM] = instr_meter_mhm_exec,
8149         [INSTR_METER_MHI] = instr_meter_mhi_exec,
8150         [INSTR_METER_MMM] = instr_meter_mmm_exec,
8151         [INSTR_METER_MMI] = instr_meter_mmi_exec,
8152         [INSTR_METER_IHM] = instr_meter_ihm_exec,
8153         [INSTR_METER_IHI] = instr_meter_ihi_exec,
8154         [INSTR_METER_IMM] = instr_meter_imm_exec,
8155         [INSTR_METER_IMI] = instr_meter_imi_exec,
8156
8157         [INSTR_TABLE] = instr_table_exec,
8158         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
8159         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
8160
8161         [INSTR_JMP] = instr_jmp_exec,
8162         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
8163         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
8164         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
8165         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
8166         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
8167         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
8168
8169         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
8170         [INSTR_JMP_EQ_S] = instr_jmp_eq_s_exec,
8171         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
8172
8173         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
8174         [INSTR_JMP_NEQ_S] = instr_jmp_neq_s_exec,
8175         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
8176
8177         [INSTR_JMP_LT] = instr_jmp_lt_exec,
8178         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
8179         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
8180         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
8181         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
8182         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
8183
8184         [INSTR_JMP_GT] = instr_jmp_gt_exec,
8185         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
8186         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
8187         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
8188         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
8189         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
8190
8191         [INSTR_RETURN] = instr_return_exec,
8192 };
8193
8194 static inline void
8195 instr_exec(struct rte_swx_pipeline *p)
8196 {
8197         struct thread *t = &p->threads[p->thread_id];
8198         struct instruction *ip = t->ip;
8199         instr_exec_t instr = instruction_table[ip->type];
8200
8201         instr(p);
8202 }
8203
8204 /*
8205  * Action.
8206  */
8207 static struct action *
8208 action_find(struct rte_swx_pipeline *p, const char *name)
8209 {
8210         struct action *elem;
8211
8212         if (!name)
8213                 return NULL;
8214
8215         TAILQ_FOREACH(elem, &p->actions, node)
8216                 if (strcmp(elem->name, name) == 0)
8217                         return elem;
8218
8219         return NULL;
8220 }
8221
8222 static struct action *
8223 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8224 {
8225         struct action *action = NULL;
8226
8227         TAILQ_FOREACH(action, &p->actions, node)
8228                 if (action->id == id)
8229                         return action;
8230
8231         return NULL;
8232 }
8233
8234 static struct field *
8235 action_field_find(struct action *a, const char *name)
8236 {
8237         return a->st ? struct_type_field_find(a->st, name) : NULL;
8238 }
8239
8240 static struct field *
8241 action_field_parse(struct action *action, const char *name)
8242 {
8243         if (name[0] != 't' || name[1] != '.')
8244                 return NULL;
8245
8246         return action_field_find(action, &name[2]);
8247 }
8248
8249 int
8250 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
8251                                const char *name,
8252                                const char *args_struct_type_name,
8253                                const char **instructions,
8254                                uint32_t n_instructions)
8255 {
8256         struct struct_type *args_struct_type;
8257         struct action *a;
8258         int err;
8259
8260         CHECK(p, EINVAL);
8261
8262         CHECK_NAME(name, EINVAL);
8263         CHECK(!action_find(p, name), EEXIST);
8264
8265         if (args_struct_type_name) {
8266                 CHECK_NAME(args_struct_type_name, EINVAL);
8267                 args_struct_type = struct_type_find(p, args_struct_type_name);
8268                 CHECK(args_struct_type, EINVAL);
8269         } else {
8270                 args_struct_type = NULL;
8271         }
8272
8273         /* Node allocation. */
8274         a = calloc(1, sizeof(struct action));
8275         CHECK(a, ENOMEM);
8276
8277         /* Node initialization. */
8278         strcpy(a->name, name);
8279         a->st = args_struct_type;
8280         a->id = p->n_actions;
8281
8282         /* Instruction translation. */
8283         err = instruction_config(p, a, instructions, n_instructions);
8284         if (err) {
8285                 free(a);
8286                 return err;
8287         }
8288
8289         /* Node add to tailq. */
8290         TAILQ_INSERT_TAIL(&p->actions, a, node);
8291         p->n_actions++;
8292
8293         return 0;
8294 }
8295
8296 static int
8297 action_build(struct rte_swx_pipeline *p)
8298 {
8299         struct action *action;
8300
8301         p->action_instructions = calloc(p->n_actions,
8302                                         sizeof(struct instruction *));
8303         CHECK(p->action_instructions, ENOMEM);
8304
8305         TAILQ_FOREACH(action, &p->actions, node)
8306                 p->action_instructions[action->id] = action->instructions;
8307
8308         return 0;
8309 }
8310
8311 static void
8312 action_build_free(struct rte_swx_pipeline *p)
8313 {
8314         free(p->action_instructions);
8315         p->action_instructions = NULL;
8316 }
8317
8318 static void
8319 action_free(struct rte_swx_pipeline *p)
8320 {
8321         action_build_free(p);
8322
8323         for ( ; ; ) {
8324                 struct action *action;
8325
8326                 action = TAILQ_FIRST(&p->actions);
8327                 if (!action)
8328                         break;
8329
8330                 TAILQ_REMOVE(&p->actions, action, node);
8331                 free(action->instructions);
8332                 free(action);
8333         }
8334 }
8335
8336 /*
8337  * Table.
8338  */
8339 static struct table_type *
8340 table_type_find(struct rte_swx_pipeline *p, const char *name)
8341 {
8342         struct table_type *elem;
8343
8344         TAILQ_FOREACH(elem, &p->table_types, node)
8345                 if (strcmp(elem->name, name) == 0)
8346                         return elem;
8347
8348         return NULL;
8349 }
8350
8351 static struct table_type *
8352 table_type_resolve(struct rte_swx_pipeline *p,
8353                    const char *recommended_type_name,
8354                    enum rte_swx_table_match_type match_type)
8355 {
8356         struct table_type *elem;
8357
8358         /* Only consider the recommended type if the match type is correct. */
8359         if (recommended_type_name)
8360                 TAILQ_FOREACH(elem, &p->table_types, node)
8361                         if (!strcmp(elem->name, recommended_type_name) &&
8362                             (elem->match_type == match_type))
8363                                 return elem;
8364
8365         /* Ignore the recommended type and get the first element with this match
8366          * type.
8367          */
8368         TAILQ_FOREACH(elem, &p->table_types, node)
8369                 if (elem->match_type == match_type)
8370                         return elem;
8371
8372         return NULL;
8373 }
8374
8375 static struct table *
8376 table_find(struct rte_swx_pipeline *p, const char *name)
8377 {
8378         struct table *elem;
8379
8380         TAILQ_FOREACH(elem, &p->tables, node)
8381                 if (strcmp(elem->name, name) == 0)
8382                         return elem;
8383
8384         return NULL;
8385 }
8386
8387 static struct table *
8388 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8389 {
8390         struct table *table = NULL;
8391
8392         TAILQ_FOREACH(table, &p->tables, node)
8393                 if (table->id == id)
8394                         return table;
8395
8396         return NULL;
8397 }
8398
8399 int
8400 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
8401                                      const char *name,
8402                                      enum rte_swx_table_match_type match_type,
8403                                      struct rte_swx_table_ops *ops)
8404 {
8405         struct table_type *elem;
8406
8407         CHECK(p, EINVAL);
8408
8409         CHECK_NAME(name, EINVAL);
8410         CHECK(!table_type_find(p, name), EEXIST);
8411
8412         CHECK(ops, EINVAL);
8413         CHECK(ops->create, EINVAL);
8414         CHECK(ops->lkp, EINVAL);
8415         CHECK(ops->free, EINVAL);
8416
8417         /* Node allocation. */
8418         elem = calloc(1, sizeof(struct table_type));
8419         CHECK(elem, ENOMEM);
8420
8421         /* Node initialization. */
8422         strcpy(elem->name, name);
8423         elem->match_type = match_type;
8424         memcpy(&elem->ops, ops, sizeof(*ops));
8425
8426         /* Node add to tailq. */
8427         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
8428
8429         return 0;
8430 }
8431
8432 static enum rte_swx_table_match_type
8433 table_match_type_resolve(struct rte_swx_match_field_params *fields,
8434                          uint32_t n_fields)
8435 {
8436         uint32_t i;
8437
8438         for (i = 0; i < n_fields; i++)
8439                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
8440                         break;
8441
8442         if (i == n_fields)
8443                 return RTE_SWX_TABLE_MATCH_EXACT;
8444
8445         if ((i == n_fields - 1) &&
8446             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
8447                 return RTE_SWX_TABLE_MATCH_LPM;
8448
8449         return RTE_SWX_TABLE_MATCH_WILDCARD;
8450 }
8451
8452 int
8453 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8454                               const char *name,
8455                               struct rte_swx_pipeline_table_params *params,
8456                               const char *recommended_table_type_name,
8457                               const char *args,
8458                               uint32_t size)
8459 {
8460         struct table_type *type;
8461         struct table *t;
8462         struct action *default_action;
8463         struct header *header = NULL;
8464         int is_header = 0;
8465         uint32_t offset_prev = 0, action_data_size_max = 0, i;
8466
8467         CHECK(p, EINVAL);
8468
8469         CHECK_NAME(name, EINVAL);
8470         CHECK(!table_find(p, name), EEXIST);
8471
8472         CHECK(params, EINVAL);
8473
8474         /* Match checks. */
8475         CHECK(!params->n_fields || params->fields, EINVAL);
8476         for (i = 0; i < params->n_fields; i++) {
8477                 struct rte_swx_match_field_params *field = &params->fields[i];
8478                 struct header *h;
8479                 struct field *hf, *mf;
8480                 uint32_t offset;
8481
8482                 CHECK_NAME(field->name, EINVAL);
8483
8484                 hf = header_field_parse(p, field->name, &h);
8485                 mf = metadata_field_parse(p, field->name);
8486                 CHECK(hf || mf, EINVAL);
8487
8488                 offset = hf ? hf->offset : mf->offset;
8489
8490                 if (i == 0) {
8491                         is_header = hf ? 1 : 0;
8492                         header = hf ? h : NULL;
8493                         offset_prev = offset;
8494
8495                         continue;
8496                 }
8497
8498                 CHECK((is_header && hf && (h->id == header->id)) ||
8499                       (!is_header && mf), EINVAL);
8500
8501                 CHECK(offset > offset_prev, EINVAL);
8502                 offset_prev = offset;
8503         }
8504
8505         /* Action checks. */
8506         CHECK(params->n_actions, EINVAL);
8507         CHECK(params->action_names, EINVAL);
8508         for (i = 0; i < params->n_actions; i++) {
8509                 const char *action_name = params->action_names[i];
8510                 struct action *a;
8511                 uint32_t action_data_size;
8512
8513                 CHECK_NAME(action_name, EINVAL);
8514
8515                 a = action_find(p, action_name);
8516                 CHECK(a, EINVAL);
8517
8518                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8519                 if (action_data_size > action_data_size_max)
8520                         action_data_size_max = action_data_size;
8521         }
8522
8523         CHECK_NAME(params->default_action_name, EINVAL);
8524         for (i = 0; i < p->n_actions; i++)
8525                 if (!strcmp(params->action_names[i],
8526                             params->default_action_name))
8527                         break;
8528         CHECK(i < params->n_actions, EINVAL);
8529         default_action = action_find(p, params->default_action_name);
8530         CHECK((default_action->st && params->default_action_data) ||
8531               !params->default_action_data, EINVAL);
8532
8533         /* Table type checks. */
8534         if (recommended_table_type_name)
8535                 CHECK_NAME(recommended_table_type_name, EINVAL);
8536
8537         if (params->n_fields) {
8538                 enum rte_swx_table_match_type match_type;
8539
8540                 match_type = table_match_type_resolve(params->fields,
8541                                                       params->n_fields);
8542                 type = table_type_resolve(p,
8543                                           recommended_table_type_name,
8544                                           match_type);
8545                 CHECK(type, EINVAL);
8546         } else {
8547                 type = NULL;
8548         }
8549
8550         /* Memory allocation. */
8551         t = calloc(1, sizeof(struct table));
8552         CHECK(t, ENOMEM);
8553
8554         t->fields = calloc(params->n_fields, sizeof(struct match_field));
8555         if (!t->fields) {
8556                 free(t);
8557                 CHECK(0, ENOMEM);
8558         }
8559
8560         t->actions = calloc(params->n_actions, sizeof(struct action *));
8561         if (!t->actions) {
8562                 free(t->fields);
8563                 free(t);
8564                 CHECK(0, ENOMEM);
8565         }
8566
8567         if (action_data_size_max) {
8568                 t->default_action_data = calloc(1, action_data_size_max);
8569                 if (!t->default_action_data) {
8570                         free(t->actions);
8571                         free(t->fields);
8572                         free(t);
8573                         CHECK(0, ENOMEM);
8574                 }
8575         }
8576
8577         /* Node initialization. */
8578         strcpy(t->name, name);
8579         if (args && args[0])
8580                 strcpy(t->args, args);
8581         t->type = type;
8582
8583         for (i = 0; i < params->n_fields; i++) {
8584                 struct rte_swx_match_field_params *field = &params->fields[i];
8585                 struct match_field *f = &t->fields[i];
8586
8587                 f->match_type = field->match_type;
8588                 f->field = is_header ?
8589                         header_field_parse(p, field->name, NULL) :
8590                         metadata_field_parse(p, field->name);
8591         }
8592         t->n_fields = params->n_fields;
8593         t->is_header = is_header;
8594         t->header = header;
8595
8596         for (i = 0; i < params->n_actions; i++)
8597                 t->actions[i] = action_find(p, params->action_names[i]);
8598         t->default_action = default_action;
8599         if (default_action->st)
8600                 memcpy(t->default_action_data,
8601                        params->default_action_data,
8602                        default_action->st->n_bits / 8);
8603         t->n_actions = params->n_actions;
8604         t->default_action_is_const = params->default_action_is_const;
8605         t->action_data_size_max = action_data_size_max;
8606
8607         t->size = size;
8608         t->id = p->n_tables;
8609
8610         /* Node add to tailq. */
8611         TAILQ_INSERT_TAIL(&p->tables, t, node);
8612         p->n_tables++;
8613
8614         return 0;
8615 }
8616
8617 static struct rte_swx_table_params *
8618 table_params_get(struct table *table)
8619 {
8620         struct rte_swx_table_params *params;
8621         struct field *first, *last;
8622         uint8_t *key_mask;
8623         uint32_t key_size, key_offset, action_data_size, i;
8624
8625         /* Memory allocation. */
8626         params = calloc(1, sizeof(struct rte_swx_table_params));
8627         if (!params)
8628                 return NULL;
8629
8630         /* Key offset and size. */
8631         first = table->fields[0].field;
8632         last = table->fields[table->n_fields - 1].field;
8633         key_offset = first->offset / 8;
8634         key_size = (last->offset + last->n_bits - first->offset) / 8;
8635
8636         /* Memory allocation. */
8637         key_mask = calloc(1, key_size);
8638         if (!key_mask) {
8639                 free(params);
8640                 return NULL;
8641         }
8642
8643         /* Key mask. */
8644         for (i = 0; i < table->n_fields; i++) {
8645                 struct field *f = table->fields[i].field;
8646                 uint32_t start = (f->offset - first->offset) / 8;
8647                 size_t size = f->n_bits / 8;
8648
8649                 memset(&key_mask[start], 0xFF, size);
8650         }
8651
8652         /* Action data size. */
8653         action_data_size = 0;
8654         for (i = 0; i < table->n_actions; i++) {
8655                 struct action *action = table->actions[i];
8656                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8657
8658                 if (ads > action_data_size)
8659                         action_data_size = ads;
8660         }
8661
8662         /* Fill in. */
8663         params->match_type = table->type->match_type;
8664         params->key_size = key_size;
8665         params->key_offset = key_offset;
8666         params->key_mask0 = key_mask;
8667         params->action_data_size = action_data_size;
8668         params->n_keys_max = table->size;
8669
8670         return params;
8671 }
8672
8673 static void
8674 table_params_free(struct rte_swx_table_params *params)
8675 {
8676         if (!params)
8677                 return;
8678
8679         free(params->key_mask0);
8680         free(params);
8681 }
8682
8683 static int
8684 table_state_build(struct rte_swx_pipeline *p)
8685 {
8686         struct table *table;
8687
8688         p->table_state = calloc(p->n_tables,
8689                                 sizeof(struct rte_swx_table_state));
8690         CHECK(p->table_state, ENOMEM);
8691
8692         TAILQ_FOREACH(table, &p->tables, node) {
8693                 struct rte_swx_table_state *ts = &p->table_state[table->id];
8694
8695                 if (table->type) {
8696                         struct rte_swx_table_params *params;
8697
8698                         /* ts->obj. */
8699                         params = table_params_get(table);
8700                         CHECK(params, ENOMEM);
8701
8702                         ts->obj = table->type->ops.create(params,
8703                                 NULL,
8704                                 table->args,
8705                                 p->numa_node);
8706
8707                         table_params_free(params);
8708                         CHECK(ts->obj, ENODEV);
8709                 }
8710
8711                 /* ts->default_action_data. */
8712                 if (table->action_data_size_max) {
8713                         ts->default_action_data =
8714                                 malloc(table->action_data_size_max);
8715                         CHECK(ts->default_action_data, ENOMEM);
8716
8717                         memcpy(ts->default_action_data,
8718                                table->default_action_data,
8719                                table->action_data_size_max);
8720                 }
8721
8722                 /* ts->default_action_id. */
8723                 ts->default_action_id = table->default_action->id;
8724         }
8725
8726         return 0;
8727 }
8728
8729 static void
8730 table_state_build_free(struct rte_swx_pipeline *p)
8731 {
8732         uint32_t i;
8733
8734         if (!p->table_state)
8735                 return;
8736
8737         for (i = 0; i < p->n_tables; i++) {
8738                 struct rte_swx_table_state *ts = &p->table_state[i];
8739                 struct table *table = table_find_by_id(p, i);
8740
8741                 /* ts->obj. */
8742                 if (table->type && ts->obj)
8743                         table->type->ops.free(ts->obj);
8744
8745                 /* ts->default_action_data. */
8746                 free(ts->default_action_data);
8747         }
8748
8749         free(p->table_state);
8750         p->table_state = NULL;
8751 }
8752
8753 static void
8754 table_state_free(struct rte_swx_pipeline *p)
8755 {
8756         table_state_build_free(p);
8757 }
8758
8759 static int
8760 table_stub_lkp(void *table __rte_unused,
8761                void *mailbox __rte_unused,
8762                uint8_t **key __rte_unused,
8763                uint64_t *action_id __rte_unused,
8764                uint8_t **action_data __rte_unused,
8765                int *hit)
8766 {
8767         *hit = 0;
8768         return 1; /* DONE. */
8769 }
8770
8771 static int
8772 table_build(struct rte_swx_pipeline *p)
8773 {
8774         uint32_t i;
8775
8776         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8777                 struct thread *t = &p->threads[i];
8778                 struct table *table;
8779
8780                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8781                 CHECK(t->tables, ENOMEM);
8782
8783                 TAILQ_FOREACH(table, &p->tables, node) {
8784                         struct table_runtime *r = &t->tables[table->id];
8785
8786                         if (table->type) {
8787                                 uint64_t size;
8788
8789                                 size = table->type->ops.mailbox_size_get();
8790
8791                                 /* r->func. */
8792                                 r->func = table->type->ops.lkp;
8793
8794                                 /* r->mailbox. */
8795                                 if (size) {
8796                                         r->mailbox = calloc(1, size);
8797                                         CHECK(r->mailbox, ENOMEM);
8798                                 }
8799
8800                                 /* r->key. */
8801                                 r->key = table->is_header ?
8802                                         &t->structs[table->header->struct_id] :
8803                                         &t->structs[p->metadata_struct_id];
8804                         } else {
8805                                 r->func = table_stub_lkp;
8806                         }
8807                 }
8808         }
8809
8810         return 0;
8811 }
8812
8813 static void
8814 table_build_free(struct rte_swx_pipeline *p)
8815 {
8816         uint32_t i;
8817
8818         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8819                 struct thread *t = &p->threads[i];
8820                 uint32_t j;
8821
8822                 if (!t->tables)
8823                         continue;
8824
8825                 for (j = 0; j < p->n_tables; j++) {
8826                         struct table_runtime *r = &t->tables[j];
8827
8828                         free(r->mailbox);
8829                 }
8830
8831                 free(t->tables);
8832                 t->tables = NULL;
8833         }
8834 }
8835
8836 static void
8837 table_free(struct rte_swx_pipeline *p)
8838 {
8839         table_build_free(p);
8840
8841         /* Tables. */
8842         for ( ; ; ) {
8843                 struct table *elem;
8844
8845                 elem = TAILQ_FIRST(&p->tables);
8846                 if (!elem)
8847                         break;
8848
8849                 TAILQ_REMOVE(&p->tables, elem, node);
8850                 free(elem->fields);
8851                 free(elem->actions);
8852                 free(elem->default_action_data);
8853                 free(elem);
8854         }
8855
8856         /* Table types. */
8857         for ( ; ; ) {
8858                 struct table_type *elem;
8859
8860                 elem = TAILQ_FIRST(&p->table_types);
8861                 if (!elem)
8862                         break;
8863
8864                 TAILQ_REMOVE(&p->table_types, elem, node);
8865                 free(elem);
8866         }
8867 }
8868
8869 /*
8870  * Register array.
8871  */
8872 static struct regarray *
8873 regarray_find(struct rte_swx_pipeline *p, const char *name)
8874 {
8875         struct regarray *elem;
8876
8877         TAILQ_FOREACH(elem, &p->regarrays, node)
8878                 if (!strcmp(elem->name, name))
8879                         return elem;
8880
8881         return NULL;
8882 }
8883
8884 static struct regarray *
8885 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8886 {
8887         struct regarray *elem = NULL;
8888
8889         TAILQ_FOREACH(elem, &p->regarrays, node)
8890                 if (elem->id == id)
8891                         return elem;
8892
8893         return NULL;
8894 }
8895
8896 int
8897 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8898                               const char *name,
8899                               uint32_t size,
8900                               uint64_t init_val)
8901 {
8902         struct regarray *r;
8903
8904         CHECK(p, EINVAL);
8905
8906         CHECK_NAME(name, EINVAL);
8907         CHECK(!regarray_find(p, name), EEXIST);
8908
8909         CHECK(size, EINVAL);
8910         size = rte_align32pow2(size);
8911
8912         /* Memory allocation. */
8913         r = calloc(1, sizeof(struct regarray));
8914         CHECK(r, ENOMEM);
8915
8916         /* Node initialization. */
8917         strcpy(r->name, name);
8918         r->init_val = init_val;
8919         r->size = size;
8920         r->id = p->n_regarrays;
8921
8922         /* Node add to tailq. */
8923         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8924         p->n_regarrays++;
8925
8926         return 0;
8927 }
8928
8929 static int
8930 regarray_build(struct rte_swx_pipeline *p)
8931 {
8932         struct regarray *regarray;
8933
8934         if (!p->n_regarrays)
8935                 return 0;
8936
8937         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8938         CHECK(p->regarray_runtime, ENOMEM);
8939
8940         TAILQ_FOREACH(regarray, &p->regarrays, node) {
8941                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8942                 uint32_t i;
8943
8944                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8945                                          RTE_CACHE_LINE_SIZE,
8946                                          p->numa_node);
8947                 CHECK(r->regarray, ENOMEM);
8948
8949                 if (regarray->init_val)
8950                         for (i = 0; i < regarray->size; i++)
8951                                 r->regarray[i] = regarray->init_val;
8952
8953                 r->size_mask = regarray->size - 1;
8954         }
8955
8956         return 0;
8957 }
8958
8959 static void
8960 regarray_build_free(struct rte_swx_pipeline *p)
8961 {
8962         uint32_t i;
8963
8964         if (!p->regarray_runtime)
8965                 return;
8966
8967         for (i = 0; i < p->n_regarrays; i++) {
8968                 struct regarray *regarray = regarray_find_by_id(p, i);
8969                 struct regarray_runtime *r = &p->regarray_runtime[i];
8970
8971                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8972         }
8973
8974         free(p->regarray_runtime);
8975         p->regarray_runtime = NULL;
8976 }
8977
8978 static void
8979 regarray_free(struct rte_swx_pipeline *p)
8980 {
8981         regarray_build_free(p);
8982
8983         for ( ; ; ) {
8984                 struct regarray *elem;
8985
8986                 elem = TAILQ_FIRST(&p->regarrays);
8987                 if (!elem)
8988                         break;
8989
8990                 TAILQ_REMOVE(&p->regarrays, elem, node);
8991                 free(elem);
8992         }
8993 }
8994
8995 /*
8996  * Meter array.
8997  */
8998 static struct meter_profile *
8999 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9000 {
9001         struct meter_profile *elem;
9002
9003         TAILQ_FOREACH(elem, &p->meter_profiles, node)
9004                 if (!strcmp(elem->name, name))
9005                         return elem;
9006
9007         return NULL;
9008 }
9009
9010 static struct metarray *
9011 metarray_find(struct rte_swx_pipeline *p, const char *name)
9012 {
9013         struct metarray *elem;
9014
9015         TAILQ_FOREACH(elem, &p->metarrays, node)
9016                 if (!strcmp(elem->name, name))
9017                         return elem;
9018
9019         return NULL;
9020 }
9021
9022 static struct metarray *
9023 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9024 {
9025         struct metarray *elem = NULL;
9026
9027         TAILQ_FOREACH(elem, &p->metarrays, node)
9028                 if (elem->id == id)
9029                         return elem;
9030
9031         return NULL;
9032 }
9033
9034 int
9035 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9036                                  const char *name,
9037                                  uint32_t size)
9038 {
9039         struct metarray *m;
9040
9041         CHECK(p, EINVAL);
9042
9043         CHECK_NAME(name, EINVAL);
9044         CHECK(!metarray_find(p, name), EEXIST);
9045
9046         CHECK(size, EINVAL);
9047         size = rte_align32pow2(size);
9048
9049         /* Memory allocation. */
9050         m = calloc(1, sizeof(struct metarray));
9051         CHECK(m, ENOMEM);
9052
9053         /* Node initialization. */
9054         strcpy(m->name, name);
9055         m->size = size;
9056         m->id = p->n_metarrays;
9057
9058         /* Node add to tailq. */
9059         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9060         p->n_metarrays++;
9061
9062         return 0;
9063 }
9064
9065 struct meter_profile meter_profile_default = {
9066         .node = {0},
9067         .name = "",
9068         .params = {0},
9069
9070         .profile = {
9071                 .cbs = 10000,
9072                 .pbs = 10000,
9073                 .cir_period = 1,
9074                 .cir_bytes_per_period = 1,
9075                 .pir_period = 1,
9076                 .pir_bytes_per_period = 1,
9077         },
9078
9079         .n_users = 0,
9080 };
9081
9082 static void
9083 meter_init(struct meter *m)
9084 {
9085         memset(m, 0, sizeof(struct meter));
9086         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9087         m->profile = &meter_profile_default;
9088         m->color_mask = RTE_COLOR_GREEN;
9089
9090         meter_profile_default.n_users++;
9091 }
9092
9093 static int
9094 metarray_build(struct rte_swx_pipeline *p)
9095 {
9096         struct metarray *m;
9097
9098         if (!p->n_metarrays)
9099                 return 0;
9100
9101         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9102         CHECK(p->metarray_runtime, ENOMEM);
9103
9104         TAILQ_FOREACH(m, &p->metarrays, node) {
9105                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9106                 uint32_t i;
9107
9108                 r->metarray = env_malloc(m->size * sizeof(struct meter),
9109                                          RTE_CACHE_LINE_SIZE,
9110                                          p->numa_node);
9111                 CHECK(r->metarray, ENOMEM);
9112
9113                 for (i = 0; i < m->size; i++)
9114                         meter_init(&r->metarray[i]);
9115
9116                 r->size_mask = m->size - 1;
9117         }
9118
9119         return 0;
9120 }
9121
9122 static void
9123 metarray_build_free(struct rte_swx_pipeline *p)
9124 {
9125         uint32_t i;
9126
9127         if (!p->metarray_runtime)
9128                 return;
9129
9130         for (i = 0; i < p->n_metarrays; i++) {
9131                 struct metarray *m = metarray_find_by_id(p, i);
9132                 struct metarray_runtime *r = &p->metarray_runtime[i];
9133
9134                 env_free(r->metarray, m->size * sizeof(struct meter));
9135         }
9136
9137         free(p->metarray_runtime);
9138         p->metarray_runtime = NULL;
9139 }
9140
9141 static void
9142 metarray_free(struct rte_swx_pipeline *p)
9143 {
9144         metarray_build_free(p);
9145
9146         /* Meter arrays. */
9147         for ( ; ; ) {
9148                 struct metarray *elem;
9149
9150                 elem = TAILQ_FIRST(&p->metarrays);
9151                 if (!elem)
9152                         break;
9153
9154                 TAILQ_REMOVE(&p->metarrays, elem, node);
9155                 free(elem);
9156         }
9157
9158         /* Meter profiles. */
9159         for ( ; ; ) {
9160                 struct meter_profile *elem;
9161
9162                 elem = TAILQ_FIRST(&p->meter_profiles);
9163                 if (!elem)
9164                         break;
9165
9166                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9167                 free(elem);
9168         }
9169 }
9170
9171 /*
9172  * Pipeline.
9173  */
9174 int
9175 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9176 {
9177         struct rte_swx_pipeline *pipeline;
9178
9179         /* Check input parameters. */
9180         CHECK(p, EINVAL);
9181
9182         /* Memory allocation. */
9183         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9184         CHECK(pipeline, ENOMEM);
9185
9186         /* Initialization. */
9187         TAILQ_INIT(&pipeline->struct_types);
9188         TAILQ_INIT(&pipeline->port_in_types);
9189         TAILQ_INIT(&pipeline->ports_in);
9190         TAILQ_INIT(&pipeline->port_out_types);
9191         TAILQ_INIT(&pipeline->ports_out);
9192         TAILQ_INIT(&pipeline->extern_types);
9193         TAILQ_INIT(&pipeline->extern_objs);
9194         TAILQ_INIT(&pipeline->extern_funcs);
9195         TAILQ_INIT(&pipeline->headers);
9196         TAILQ_INIT(&pipeline->actions);
9197         TAILQ_INIT(&pipeline->table_types);
9198         TAILQ_INIT(&pipeline->tables);
9199         TAILQ_INIT(&pipeline->regarrays);
9200         TAILQ_INIT(&pipeline->meter_profiles);
9201         TAILQ_INIT(&pipeline->metarrays);
9202
9203         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9204         pipeline->numa_node = numa_node;
9205
9206         *p = pipeline;
9207         return 0;
9208 }
9209
9210 void
9211 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9212 {
9213         if (!p)
9214                 return;
9215
9216         free(p->instructions);
9217
9218         metarray_free(p);
9219         regarray_free(p);
9220         table_state_free(p);
9221         table_free(p);
9222         action_free(p);
9223         metadata_free(p);
9224         header_free(p);
9225         extern_func_free(p);
9226         extern_obj_free(p);
9227         port_out_free(p);
9228         port_in_free(p);
9229         struct_free(p);
9230
9231         free(p);
9232 }
9233
9234 int
9235 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9236                                      const char **instructions,
9237                                      uint32_t n_instructions)
9238 {
9239         int err;
9240         uint32_t i;
9241
9242         err = instruction_config(p, NULL, instructions, n_instructions);
9243         if (err)
9244                 return err;
9245
9246         /* Thread instruction pointer reset. */
9247         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9248                 struct thread *t = &p->threads[i];
9249
9250                 thread_ip_reset(p, t);
9251         }
9252
9253         return 0;
9254 }
9255
9256 int
9257 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9258 {
9259         int status;
9260
9261         CHECK(p, EINVAL);
9262         CHECK(p->build_done == 0, EEXIST);
9263
9264         status = port_in_build(p);
9265         if (status)
9266                 goto error;
9267
9268         status = port_out_build(p);
9269         if (status)
9270                 goto error;
9271
9272         status = struct_build(p);
9273         if (status)
9274                 goto error;
9275
9276         status = extern_obj_build(p);
9277         if (status)
9278                 goto error;
9279
9280         status = extern_func_build(p);
9281         if (status)
9282                 goto error;
9283
9284         status = header_build(p);
9285         if (status)
9286                 goto error;
9287
9288         status = metadata_build(p);
9289         if (status)
9290                 goto error;
9291
9292         status = action_build(p);
9293         if (status)
9294                 goto error;
9295
9296         status = table_build(p);
9297         if (status)
9298                 goto error;
9299
9300         status = table_state_build(p);
9301         if (status)
9302                 goto error;
9303
9304         status = regarray_build(p);
9305         if (status)
9306                 goto error;
9307
9308         status = metarray_build(p);
9309         if (status)
9310                 goto error;
9311
9312         p->build_done = 1;
9313         return 0;
9314
9315 error:
9316         metarray_build_free(p);
9317         regarray_build_free(p);
9318         table_state_build_free(p);
9319         table_build_free(p);
9320         action_build_free(p);
9321         metadata_build_free(p);
9322         header_build_free(p);
9323         extern_func_build_free(p);
9324         extern_obj_build_free(p);
9325         port_out_build_free(p);
9326         port_in_build_free(p);
9327         struct_build_free(p);
9328
9329         return status;
9330 }
9331
9332 void
9333 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9334 {
9335         uint32_t i;
9336
9337         for (i = 0; i < n_instructions; i++)
9338                 instr_exec(p);
9339 }
9340
9341 void
9342 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9343 {
9344         uint32_t i;
9345
9346         for (i = 0; i < p->n_ports_out; i++) {
9347                 struct port_out_runtime *port = &p->out[i];
9348
9349                 if (port->flush)
9350                         port->flush(port->obj);
9351         }
9352 }
9353
9354 /*
9355  * Control.
9356  */
9357 int
9358 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9359                               struct rte_swx_ctl_pipeline_info *pipeline)
9360 {
9361         struct action *action;
9362         struct table *table;
9363         uint32_t n_actions = 0, n_tables = 0;
9364
9365         if (!p || !pipeline)
9366                 return -EINVAL;
9367
9368         TAILQ_FOREACH(action, &p->actions, node)
9369                 n_actions++;
9370
9371         TAILQ_FOREACH(table, &p->tables, node)
9372                 n_tables++;
9373
9374         pipeline->n_ports_in = p->n_ports_in;
9375         pipeline->n_ports_out = p->n_ports_out;
9376         pipeline->n_actions = n_actions;
9377         pipeline->n_tables = n_tables;
9378         pipeline->n_regarrays = p->n_regarrays;
9379         pipeline->n_metarrays = p->n_metarrays;
9380
9381         return 0;
9382 }
9383
9384 int
9385 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9386 {
9387         if (!p || !numa_node)
9388                 return -EINVAL;
9389
9390         *numa_node = p->numa_node;
9391         return 0;
9392 }
9393
9394 int
9395 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9396                             uint32_t action_id,
9397                             struct rte_swx_ctl_action_info *action)
9398 {
9399         struct action *a = NULL;
9400
9401         if (!p || (action_id >= p->n_actions) || !action)
9402                 return -EINVAL;
9403
9404         a = action_find_by_id(p, action_id);
9405         if (!a)
9406                 return -EINVAL;
9407
9408         strcpy(action->name, a->name);
9409         action->n_args = a->st ? a->st->n_fields : 0;
9410         return 0;
9411 }
9412
9413 int
9414 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9415                                 uint32_t action_id,
9416                                 uint32_t action_arg_id,
9417                                 struct rte_swx_ctl_action_arg_info *action_arg)
9418 {
9419         struct action *a = NULL;
9420         struct field *arg = NULL;
9421
9422         if (!p || (action_id >= p->n_actions) || !action_arg)
9423                 return -EINVAL;
9424
9425         a = action_find_by_id(p, action_id);
9426         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9427                 return -EINVAL;
9428
9429         arg = &a->st->fields[action_arg_id];
9430         strcpy(action_arg->name, arg->name);
9431         action_arg->n_bits = arg->n_bits;
9432
9433         return 0;
9434 }
9435
9436 int
9437 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9438                            uint32_t table_id,
9439                            struct rte_swx_ctl_table_info *table)
9440 {
9441         struct table *t = NULL;
9442
9443         if (!p || !table)
9444                 return -EINVAL;
9445
9446         t = table_find_by_id(p, table_id);
9447         if (!t)
9448                 return -EINVAL;
9449
9450         strcpy(table->name, t->name);
9451         strcpy(table->args, t->args);
9452         table->n_match_fields = t->n_fields;
9453         table->n_actions = t->n_actions;
9454         table->default_action_is_const = t->default_action_is_const;
9455         table->size = t->size;
9456         return 0;
9457 }
9458
9459 int
9460 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9461         uint32_t table_id,
9462         uint32_t match_field_id,
9463         struct rte_swx_ctl_table_match_field_info *match_field)
9464 {
9465         struct table *t;
9466         struct match_field *f;
9467
9468         if (!p || (table_id >= p->n_tables) || !match_field)
9469                 return -EINVAL;
9470
9471         t = table_find_by_id(p, table_id);
9472         if (!t || (match_field_id >= t->n_fields))
9473                 return -EINVAL;
9474
9475         f = &t->fields[match_field_id];
9476         match_field->match_type = f->match_type;
9477         match_field->is_header = t->is_header;
9478         match_field->n_bits = f->field->n_bits;
9479         match_field->offset = f->field->offset;
9480
9481         return 0;
9482 }
9483
9484 int
9485 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9486         uint32_t table_id,
9487         uint32_t table_action_id,
9488         struct rte_swx_ctl_table_action_info *table_action)
9489 {
9490         struct table *t;
9491
9492         if (!p || (table_id >= p->n_tables) || !table_action)
9493                 return -EINVAL;
9494
9495         t = table_find_by_id(p, table_id);
9496         if (!t || (table_action_id >= t->n_actions))
9497                 return -EINVAL;
9498
9499         table_action->action_id = t->actions[table_action_id]->id;
9500
9501         return 0;
9502 }
9503
9504 int
9505 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9506                           uint32_t table_id,
9507                           struct rte_swx_table_ops *table_ops,
9508                           int *is_stub)
9509 {
9510         struct table *t;
9511
9512         if (!p || (table_id >= p->n_tables))
9513                 return -EINVAL;
9514
9515         t = table_find_by_id(p, table_id);
9516         if (!t)
9517                 return -EINVAL;
9518
9519         if (t->type) {
9520                 if (table_ops)
9521                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9522                 *is_stub = 0;
9523         } else {
9524                 *is_stub = 1;
9525         }
9526
9527         return 0;
9528 }
9529
9530 int
9531 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9532                                  struct rte_swx_table_state **table_state)
9533 {
9534         if (!p || !table_state || !p->build_done)
9535                 return -EINVAL;
9536
9537         *table_state = p->table_state;
9538         return 0;
9539 }
9540
9541 int
9542 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9543                                  struct rte_swx_table_state *table_state)
9544 {
9545         if (!p || !table_state || !p->build_done)
9546                 return -EINVAL;
9547
9548         p->table_state = table_state;
9549         return 0;
9550 }
9551
9552 int
9553 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9554                                         uint32_t port_id,
9555                                         struct rte_swx_port_in_stats *stats)
9556 {
9557         struct port_in *port;
9558
9559         if (!p || !stats)
9560                 return -EINVAL;
9561
9562         port = port_in_find(p, port_id);
9563         if (!port)
9564                 return -EINVAL;
9565
9566         port->type->ops.stats_read(port->obj, stats);
9567         return 0;
9568 }
9569
9570 int
9571 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9572                                          uint32_t port_id,
9573                                          struct rte_swx_port_out_stats *stats)
9574 {
9575         struct port_out *port;
9576
9577         if (!p || !stats)
9578                 return -EINVAL;
9579
9580         port = port_out_find(p, port_id);
9581         if (!port)
9582                 return -EINVAL;
9583
9584         port->type->ops.stats_read(port->obj, stats);
9585         return 0;
9586 }
9587
9588 int
9589 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9590                               uint32_t regarray_id,
9591                               struct rte_swx_ctl_regarray_info *regarray)
9592 {
9593         struct regarray *r;
9594
9595         if (!p || !regarray)
9596                 return -EINVAL;
9597
9598         r = regarray_find_by_id(p, regarray_id);
9599         if (!r)
9600                 return -EINVAL;
9601
9602         strcpy(regarray->name, r->name);
9603         regarray->size = r->size;
9604         return 0;
9605 }
9606
9607 int
9608 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9609                                    const char *regarray_name,
9610                                    uint32_t regarray_index,
9611                                    uint64_t *value)
9612 {
9613         struct regarray *regarray;
9614         struct regarray_runtime *r;
9615
9616         if (!p || !regarray_name || !value)
9617                 return -EINVAL;
9618
9619         regarray = regarray_find(p, regarray_name);
9620         if (!regarray || (regarray_index >= regarray->size))
9621                 return -EINVAL;
9622
9623         r = &p->regarray_runtime[regarray->id];
9624         *value = r->regarray[regarray_index];
9625         return 0;
9626 }
9627
9628 int
9629 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9630                                    const char *regarray_name,
9631                                    uint32_t regarray_index,
9632                                    uint64_t value)
9633 {
9634         struct regarray *regarray;
9635         struct regarray_runtime *r;
9636
9637         if (!p || !regarray_name)
9638                 return -EINVAL;
9639
9640         regarray = regarray_find(p, regarray_name);
9641         if (!regarray || (regarray_index >= regarray->size))
9642                 return -EINVAL;
9643
9644         r = &p->regarray_runtime[regarray->id];
9645         r->regarray[regarray_index] = value;
9646         return 0;
9647 }
9648
9649 int
9650 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9651                               uint32_t metarray_id,
9652                               struct rte_swx_ctl_metarray_info *metarray)
9653 {
9654         struct metarray *m;
9655
9656         if (!p || !metarray)
9657                 return -EINVAL;
9658
9659         m = metarray_find_by_id(p, metarray_id);
9660         if (!m)
9661                 return -EINVAL;
9662
9663         strcpy(metarray->name, m->name);
9664         metarray->size = m->size;
9665         return 0;
9666 }
9667
9668 int
9669 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9670                               const char *name,
9671                               struct rte_meter_trtcm_params *params)
9672 {
9673         struct meter_profile *mp;
9674         int status;
9675
9676         CHECK(p, EINVAL);
9677         CHECK_NAME(name, EINVAL);
9678         CHECK(params, EINVAL);
9679         CHECK(!meter_profile_find(p, name), EEXIST);
9680
9681         /* Node allocation. */
9682         mp = calloc(1, sizeof(struct meter_profile));
9683         CHECK(mp, ENOMEM);
9684
9685         /* Node initialization. */
9686         strcpy(mp->name, name);
9687         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9688         status = rte_meter_trtcm_profile_config(&mp->profile, params);
9689         if (status) {
9690                 free(mp);
9691                 CHECK(0, EINVAL);
9692         }
9693
9694         /* Node add to tailq. */
9695         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9696
9697         return 0;
9698 }
9699
9700 int
9701 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9702                                  const char *name)
9703 {
9704         struct meter_profile *mp;
9705
9706         CHECK(p, EINVAL);
9707         CHECK_NAME(name, EINVAL);
9708
9709         mp = meter_profile_find(p, name);
9710         CHECK(mp, EINVAL);
9711         CHECK(!mp->n_users, EBUSY);
9712
9713         /* Remove node from tailq. */
9714         TAILQ_REMOVE(&p->meter_profiles, mp, node);
9715         free(mp);
9716
9717         return 0;
9718 }
9719
9720 int
9721 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9722                         const char *metarray_name,
9723                         uint32_t metarray_index)
9724 {
9725         struct meter_profile *mp_old;
9726         struct metarray *metarray;
9727         struct metarray_runtime *metarray_runtime;
9728         struct meter *m;
9729
9730         CHECK(p, EINVAL);
9731         CHECK_NAME(metarray_name, EINVAL);
9732
9733         metarray = metarray_find(p, metarray_name);
9734         CHECK(metarray, EINVAL);
9735         CHECK(metarray_index < metarray->size, EINVAL);
9736
9737         metarray_runtime = &p->metarray_runtime[metarray->id];
9738         m = &metarray_runtime->metarray[metarray_index];
9739         mp_old = m->profile;
9740
9741         meter_init(m);
9742
9743         mp_old->n_users--;
9744
9745         return 0;
9746 }
9747
9748 int
9749 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9750                       const char *metarray_name,
9751                       uint32_t metarray_index,
9752                       const char *profile_name)
9753 {
9754         struct meter_profile *mp, *mp_old;
9755         struct metarray *metarray;
9756         struct metarray_runtime *metarray_runtime;
9757         struct meter *m;
9758
9759         CHECK(p, EINVAL);
9760         CHECK_NAME(metarray_name, EINVAL);
9761
9762         metarray = metarray_find(p, metarray_name);
9763         CHECK(metarray, EINVAL);
9764         CHECK(metarray_index < metarray->size, EINVAL);
9765
9766         mp = meter_profile_find(p, profile_name);
9767         CHECK(mp, EINVAL);
9768
9769         metarray_runtime = &p->metarray_runtime[metarray->id];
9770         m = &metarray_runtime->metarray[metarray_index];
9771         mp_old = m->profile;
9772
9773         memset(m, 0, sizeof(struct meter));
9774         rte_meter_trtcm_config(&m->m, &mp->profile);
9775         m->profile = mp;
9776         m->color_mask = RTE_COLORS;
9777
9778         mp->n_users++;
9779         mp_old->n_users--;
9780
9781         return 0;
9782 }
9783
9784 int
9785 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9786                              const char *metarray_name,
9787                              uint32_t metarray_index,
9788                              struct rte_swx_ctl_meter_stats *stats)
9789 {
9790         struct metarray *metarray;
9791         struct metarray_runtime *metarray_runtime;
9792         struct meter *m;
9793
9794         CHECK(p, EINVAL);
9795         CHECK_NAME(metarray_name, EINVAL);
9796
9797         metarray = metarray_find(p, metarray_name);
9798         CHECK(metarray, EINVAL);
9799         CHECK(metarray_index < metarray->size, EINVAL);
9800
9801         CHECK(stats, EINVAL);
9802
9803         metarray_runtime = &p->metarray_runtime[metarray->id];
9804         m = &metarray_runtime->metarray[metarray_index];
9805
9806         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9807         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9808
9809         return 0;
9810 }