lib/pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <inttypes.h>
   9 #include <sys/queue.h>
  10 #include <arpa/inet.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_prefetch.h>
  14 #include <rte_byteorder.h>
  15 #include <rte_cycles.h>
  16 #include <rte_meter.h>
  17
  18 #include <rte_swx_table_selector.h>
  19 #include <rte_swx_table_learner.h>
  20
  21 #include "rte_swx_pipeline.h"
  22 #include "rte_swx_ctl.h"
  23
  24 #define CHECK(condition, err_code)                                             \
  25 do {                                                                           \
  26         if (!(condition))                                                      \
  27                 return -(err_code);                                            \
  28 } while (0)
  29
  30 #define CHECK_NAME(name, err_code)                                             \
  31         CHECK((name) &&                                                        \
  32               (name)[0] &&                                                     \
  33               (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
  34               err_code)
  35
  36 #define CHECK_INSTRUCTION(instr, err_code)                                     \
  37         CHECK((instr) &&                                                       \
  38               (instr)[0] &&                                                    \
  39               (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
  40                RTE_SWX_INSTRUCTION_SIZE),                                      \
  41               err_code)
  42
  43 #ifndef TRACE_LEVEL
  44 #define TRACE_LEVEL 0
  45 #endif
  46
  47 #if TRACE_LEVEL
  48 #define TRACE(...) printf(__VA_ARGS__)
  49 #else
  50 #define TRACE(...)
  51 #endif
  52
  53 /*
  54  * Environment.
  55  */
  56 #define ntoh64(x) rte_be_to_cpu_64(x)
  57 #define hton64(x) rte_cpu_to_be_64(x)
  58
  59 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
  60
  61 #include <rte_malloc.h>
  62
  63 static void *
  64 env_malloc(size_t size, size_t alignment, int numa_node)
  65 {
  66         return rte_zmalloc_socket(NULL, size, alignment, numa_node);
  67 }
  68
  69 static void
  70 env_free(void *start, size_t size __rte_unused)
  71 {
  72         rte_free(start);
  73 }
  74
  75 #else
  76
  77 #include <numa.h>
  78
  79 static void *
  80 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
  81 {
  82         void *start;
  83
  84         if (numa_available() == -1)
  85                 return NULL;
  86
  87         start = numa_alloc_onnode(size, numa_node);
  88         if (!start)
  89                 return NULL;
  90
  91         memset(start, 0, size);
  92         return start;
  93 }
  94
  95 static void
  96 env_free(void *start, size_t size)
  97 {
  98         if (numa_available() == -1)
  99                 return;
 100
 101         numa_free(start, size);
 102 }
 103
 104 #endif
 105
 106 /*
 107  * Struct.
 108  */
 109 struct field {
 110         char name[RTE_SWX_NAME_SIZE];
 111         uint32_t n_bits;
 112         uint32_t offset;
 113         int var_size;
 114 };
 115
 116 struct struct_type {
 117         TAILQ_ENTRY(struct_type) node;
 118         char name[RTE_SWX_NAME_SIZE];
 119         struct field *fields;
 120         uint32_t n_fields;
 121         uint32_t n_bits;
 122         uint32_t n_bits_min;
 123         int var_size;
 124 };
 125
 126 TAILQ_HEAD(struct_type_tailq, struct_type);
 127
 128 /*
 129  * Input port.
 130  */
 131 struct port_in_type {
 132         TAILQ_ENTRY(port_in_type) node;
 133         char name[RTE_SWX_NAME_SIZE];
 134         struct rte_swx_port_in_ops ops;
 135 };
 136
 137 TAILQ_HEAD(port_in_type_tailq, port_in_type);
 138
 139 struct port_in {
 140         TAILQ_ENTRY(port_in) node;
 141         struct port_in_type *type;
 142         void *obj;
 143         uint32_t id;
 144 };
 145
 146 TAILQ_HEAD(port_in_tailq, port_in);
 147
 148 struct port_in_runtime {
 149         rte_swx_port_in_pkt_rx_t pkt_rx;
 150         void *obj;
 151 };
 152
 153 /*
 154  * Output port.
 155  */
 156 struct port_out_type {
 157         TAILQ_ENTRY(port_out_type) node;
 158         char name[RTE_SWX_NAME_SIZE];
 159         struct rte_swx_port_out_ops ops;
 160 };
 161
 162 TAILQ_HEAD(port_out_type_tailq, port_out_type);
 163
 164 struct port_out {
 165         TAILQ_ENTRY(port_out) node;
 166         struct port_out_type *type;
 167         void *obj;
 168         uint32_t id;
 169 };
 170
 171 TAILQ_HEAD(port_out_tailq, port_out);
 172
 173 struct port_out_runtime {
 174         rte_swx_port_out_pkt_tx_t pkt_tx;
 175         rte_swx_port_out_flush_t flush;
 176         void *obj;
 177 };
 178
 179 /*
 180  * Extern object.
 181  */
 182 struct extern_type_member_func {
 183         TAILQ_ENTRY(extern_type_member_func) node;
 184         char name[RTE_SWX_NAME_SIZE];
 185         rte_swx_extern_type_member_func_t func;
 186         uint32_t id;
 187 };
 188
 189 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 190
 191 struct extern_type {
 192         TAILQ_ENTRY(extern_type) node;
 193         char name[RTE_SWX_NAME_SIZE];
 194         struct struct_type *mailbox_struct_type;
 195         rte_swx_extern_type_constructor_t constructor;
 196         rte_swx_extern_type_destructor_t destructor;
 197         struct extern_type_member_func_tailq funcs;
 198         uint32_t n_funcs;
 199 };
 200
 201 TAILQ_HEAD(extern_type_tailq, extern_type);
 202
 203 struct extern_obj {
 204         TAILQ_ENTRY(extern_obj) node;
 205         char name[RTE_SWX_NAME_SIZE];
 206         struct extern_type *type;
 207         void *obj;
 208         uint32_t struct_id;
 209         uint32_t id;
 210 };
 211
 212 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 213
 214 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 215 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 216 #endif
 217
 218 struct extern_obj_runtime {
 219         void *obj;
 220         uint8_t *mailbox;
 221         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 222 };
 223
 224 /*
 225  * Extern function.
 226  */
 227 struct extern_func {
 228         TAILQ_ENTRY(extern_func) node;
 229         char name[RTE_SWX_NAME_SIZE];
 230         struct struct_type *mailbox_struct_type;
 231         rte_swx_extern_func_t func;
 232         uint32_t struct_id;
 233         uint32_t id;
 234 };
 235
 236 TAILQ_HEAD(extern_func_tailq, extern_func);
 237
 238 struct extern_func_runtime {
 239         uint8_t *mailbox;
 240         rte_swx_extern_func_t func;
 241 };
 242
 243 /*
 244  * Header.
 245  */
 246 struct header {
 247         TAILQ_ENTRY(header) node;
 248         char name[RTE_SWX_NAME_SIZE];
 249         struct struct_type *st;
 250         uint32_t struct_id;
 251         uint32_t id;
 252 };
 253
 254 TAILQ_HEAD(header_tailq, header);
 255
 256 struct header_runtime {
 257         uint8_t *ptr0;
 258         uint32_t n_bytes;
 259 };
 260
 261 struct header_out_runtime {
 262         uint8_t *ptr0;
 263         uint8_t *ptr;
 264         uint32_t n_bytes;
 265 };
 266
 267 /*
 268  * Instruction.
 269  */
 270
 271 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 272  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 273  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 274  * when transferred to packet meta-data and in NBO when transferred to packet
 275  * headers.
 276  */
 277
 278 /* Notation conventions:
 279  *    -Header field: H = h.header.field (dst/src)
 280  *    -Meta-data field: M = m.field (dst/src)
 281  *    -Extern object mailbox field: E = e.field (dst/src)
 282  *    -Extern function mailbox field: F = f.field (dst/src)
 283  *    -Table action data field: T = t.field (src only)
 284  *    -Immediate value: I = 32-bit unsigned value (src only)
 285  */
 286
 287 enum instruction_type {
 288         /* rx m.port_in */
 289         INSTR_RX,
 290
 291         /* tx port_out
 292          * port_out = MI
 293          */
 294         INSTR_TX,   /* port_out = M */
 295         INSTR_TX_I, /* port_out = I */
 296
 297         /* extract h.header */
 298         INSTR_HDR_EXTRACT,
 299         INSTR_HDR_EXTRACT2,
 300         INSTR_HDR_EXTRACT3,
 301         INSTR_HDR_EXTRACT4,
 302         INSTR_HDR_EXTRACT5,
 303         INSTR_HDR_EXTRACT6,
 304         INSTR_HDR_EXTRACT7,
 305         INSTR_HDR_EXTRACT8,
 306
 307         /* extract h.header m.last_field_size */
 308         INSTR_HDR_EXTRACT_M,
 309
 310         /* lookahead h.header */
 311         INSTR_HDR_LOOKAHEAD,
 312
 313         /* emit h.header */
 314         INSTR_HDR_EMIT,
 315         INSTR_HDR_EMIT_TX,
 316         INSTR_HDR_EMIT2_TX,
 317         INSTR_HDR_EMIT3_TX,
 318         INSTR_HDR_EMIT4_TX,
 319         INSTR_HDR_EMIT5_TX,
 320         INSTR_HDR_EMIT6_TX,
 321         INSTR_HDR_EMIT7_TX,
 322         INSTR_HDR_EMIT8_TX,
 323
 324         /* validate h.header */
 325         INSTR_HDR_VALIDATE,
 326
 327         /* invalidate h.header */
 328         INSTR_HDR_INVALIDATE,
 329
 330         /* mov dst src
 331          * dst = src
 332          * dst = HMEF, src = HMEFTI
 333          */
 334         INSTR_MOV,    /* dst = MEF, src = MEFT */
 335         INSTR_MOV_MH, /* dst = MEF, src = H */
 336         INSTR_MOV_HM, /* dst = H, src = MEFT */
 337         INSTR_MOV_HH, /* dst = H, src = H */
 338         INSTR_MOV_I,  /* dst = HMEF, src = I */
 339
 340         /* dma h.header t.field
 341          * memcpy(h.header, t.field, sizeof(h.header))
 342          */
 343         INSTR_DMA_HT,
 344         INSTR_DMA_HT2,
 345         INSTR_DMA_HT3,
 346         INSTR_DMA_HT4,
 347         INSTR_DMA_HT5,
 348         INSTR_DMA_HT6,
 349         INSTR_DMA_HT7,
 350         INSTR_DMA_HT8,
 351
 352         /* add dst src
 353          * dst += src
 354          * dst = HMEF, src = HMEFTI
 355          */
 356         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 357         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 358         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 359         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 360         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 361         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 362
 363         /* sub dst src
 364          * dst -= src
 365          * dst = HMEF, src = HMEFTI
 366          */
 367         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 368         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 369         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 370         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 371         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 372         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 373
 374         /* ckadd dst src
 375          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 376          * dst = H, src = {H, h.header}
 377          */
 378         INSTR_ALU_CKADD_FIELD,    /* src = H */
 379         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 380         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 381
 382         /* cksub dst src
 383          * dst = dst '- src
 384          * dst = H, src = H
 385          */
 386         INSTR_ALU_CKSUB_FIELD,
 387
 388         /* and dst src
 389          * dst &= src
 390          * dst = HMEF, src = HMEFTI
 391          */
 392         INSTR_ALU_AND,    /* dst = MEF, src = MEFT */
 393         INSTR_ALU_AND_MH, /* dst = MEF, src = H */
 394         INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
 395         INSTR_ALU_AND_HH, /* dst = H, src = H */
 396         INSTR_ALU_AND_I,  /* dst = HMEF, src = I */
 397
 398         /* or dst src
 399          * dst |= src
 400          * dst = HMEF, src = HMEFTI
 401          */
 402         INSTR_ALU_OR,    /* dst = MEF, src = MEFT */
 403         INSTR_ALU_OR_MH, /* dst = MEF, src = H */
 404         INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
 405         INSTR_ALU_OR_HH, /* dst = H, src = H */
 406         INSTR_ALU_OR_I,  /* dst = HMEF, src = I */
 407
 408         /* xor dst src
 409          * dst ^= src
 410          * dst = HMEF, src = HMEFTI
 411          */
 412         INSTR_ALU_XOR,    /* dst = MEF, src = MEFT */
 413         INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
 414         INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
 415         INSTR_ALU_XOR_HH, /* dst = H, src = H */
 416         INSTR_ALU_XOR_I,  /* dst = HMEF, src = I */
 417
 418         /* shl dst src
 419          * dst <<= src
 420          * dst = HMEF, src = HMEFTI
 421          */
 422         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 423         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 424         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 425         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 426         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 427         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 428
 429         /* shr dst src
 430          * dst >>= src
 431          * dst = HMEF, src = HMEFTI
 432          */
 433         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 434         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 435         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 436         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 437         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 438         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 439
 440         /* regprefetch REGARRAY index
 441          * prefetch REGARRAY[index]
 442          * index = HMEFTI
 443          */
 444         INSTR_REGPREFETCH_RH, /* index = H */
 445         INSTR_REGPREFETCH_RM, /* index = MEFT */
 446         INSTR_REGPREFETCH_RI, /* index = I */
 447
 448         /* regrd dst REGARRAY index
 449          * dst = REGARRAY[index]
 450          * dst = HMEF, index = HMEFTI
 451          */
 452         INSTR_REGRD_HRH, /* dst = H, index = H */
 453         INSTR_REGRD_HRM, /* dst = H, index = MEFT */
 454         INSTR_REGRD_HRI, /* dst = H, index = I */
 455         INSTR_REGRD_MRH, /* dst = MEF, index = H */
 456         INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
 457         INSTR_REGRD_MRI, /* dst = MEF, index = I */
 458
 459         /* regwr REGARRAY index src
 460          * REGARRAY[index] = src
 461          * index = HMEFTI, src = HMEFTI
 462          */
 463         INSTR_REGWR_RHH, /* index = H, src = H */
 464         INSTR_REGWR_RHM, /* index = H, src = MEFT */
 465         INSTR_REGWR_RHI, /* index = H, src = I */
 466         INSTR_REGWR_RMH, /* index = MEFT, src = H */
 467         INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
 468         INSTR_REGWR_RMI, /* index = MEFT, src = I */
 469         INSTR_REGWR_RIH, /* index = I, src = H */
 470         INSTR_REGWR_RIM, /* index = I, src = MEFT */
 471         INSTR_REGWR_RII, /* index = I, src = I */
 472
 473         /* regadd REGARRAY index src
 474          * REGARRAY[index] += src
 475          * index = HMEFTI, src = HMEFTI
 476          */
 477         INSTR_REGADD_RHH, /* index = H, src = H */
 478         INSTR_REGADD_RHM, /* index = H, src = MEFT */
 479         INSTR_REGADD_RHI, /* index = H, src = I */
 480         INSTR_REGADD_RMH, /* index = MEFT, src = H */
 481         INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
 482         INSTR_REGADD_RMI, /* index = MEFT, src = I */
 483         INSTR_REGADD_RIH, /* index = I, src = H */
 484         INSTR_REGADD_RIM, /* index = I, src = MEFT */
 485         INSTR_REGADD_RII, /* index = I, src = I */
 486
 487         /* metprefetch METARRAY index
 488          * prefetch METARRAY[index]
 489          * index = HMEFTI
 490          */
 491         INSTR_METPREFETCH_H, /* index = H */
 492         INSTR_METPREFETCH_M, /* index = MEFT */
 493         INSTR_METPREFETCH_I, /* index = I */
 494
 495         /* meter METARRAY index length color_in color_out
 496          * color_out = meter(METARRAY[index], length, color_in)
 497          * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
 498          */
 499         INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
 500         INSTR_METER_HHI, /* index = H, length = H, color_in = I */
 501         INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
 502         INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
 503         INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
 504         INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
 505         INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
 506         INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
 507         INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
 508         INSTR_METER_IHI, /* index = I, length = H, color_in = I */
 509         INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
 510         INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
 511
 512         /* table TABLE */
 513         INSTR_TABLE,
 514         INSTR_SELECTOR,
 515         INSTR_LEARNER,
 516
 517         /* learn LEARNER ACTION_NAME */
 518         INSTR_LEARNER_LEARN,
 519
 520         /* forget */
 521         INSTR_LEARNER_FORGET,
 522
 523         /* extern e.obj.func */
 524         INSTR_EXTERN_OBJ,
 525
 526         /* extern f.func */
 527         INSTR_EXTERN_FUNC,
 528
 529         /* jmp LABEL
 530          * Unconditional jump
 531          */
 532         INSTR_JMP,
 533
 534         /* jmpv LABEL h.header
 535          * Jump if header is valid
 536          */
 537         INSTR_JMP_VALID,
 538
 539         /* jmpnv LABEL h.header
 540          * Jump if header is invalid
 541          */
 542         INSTR_JMP_INVALID,
 543
 544         /* jmph LABEL
 545          * Jump if table lookup hit
 546          */
 547         INSTR_JMP_HIT,
 548
 549         /* jmpnh LABEL
 550          * Jump if table lookup miss
 551          */
 552         INSTR_JMP_MISS,
 553
 554         /* jmpa LABEL ACTION
 555          * Jump if action run
 556          */
 557         INSTR_JMP_ACTION_HIT,
 558
 559         /* jmpna LABEL ACTION
 560          * Jump if action not run
 561          */
 562         INSTR_JMP_ACTION_MISS,
 563
 564         /* jmpeq LABEL a b
 565          * Jump if a is equal to b
 566          * a = HMEFT, b = HMEFTI
 567          */
 568         INSTR_JMP_EQ,    /* a = MEFT, b = MEFT */
 569         INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
 570         INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
 571         INSTR_JMP_EQ_HH, /* a = H, b = H */
 572         INSTR_JMP_EQ_I,  /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 573
 574         /* jmpneq LABEL a b
 575          * Jump if a is not equal to b
 576          * a = HMEFT, b = HMEFTI
 577          */
 578         INSTR_JMP_NEQ,    /* a = MEFT, b = MEFT */
 579         INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
 580         INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
 581         INSTR_JMP_NEQ_HH, /* a = H, b = H */
 582         INSTR_JMP_NEQ_I,  /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 583
 584         /* jmplt LABEL a b
 585          * Jump if a is less than b
 586          * a = HMEFT, b = HMEFTI
 587          */
 588         INSTR_JMP_LT,    /* a = MEFT, b = MEFT */
 589         INSTR_JMP_LT_MH, /* a = MEFT, b = H */
 590         INSTR_JMP_LT_HM, /* a = H, b = MEFT */
 591         INSTR_JMP_LT_HH, /* a = H, b = H */
 592         INSTR_JMP_LT_MI, /* a = MEFT, b = I */
 593         INSTR_JMP_LT_HI, /* a = H, b = I */
 594
 595         /* jmpgt LABEL a b
 596          * Jump if a is greater than b
 597          * a = HMEFT, b = HMEFTI
 598          */
 599         INSTR_JMP_GT,    /* a = MEFT, b = MEFT */
 600         INSTR_JMP_GT_MH, /* a = MEFT, b = H */
 601         INSTR_JMP_GT_HM, /* a = H, b = MEFT */
 602         INSTR_JMP_GT_HH, /* a = H, b = H */
 603         INSTR_JMP_GT_MI, /* a = MEFT, b = I */
 604         INSTR_JMP_GT_HI, /* a = H, b = I */
 605
 606         /* return
 607          * Return from action
 608          */
 609         INSTR_RETURN,
 610 };
 611
 612 struct instr_operand {
 613         uint8_t struct_id;
 614         uint8_t n_bits;
 615         uint8_t offset;
 616         uint8_t pad;
 617 };
 618
 619 struct instr_io {
 620         struct {
 621                 union {
 622                         struct {
 623                                 uint8_t offset;
 624                                 uint8_t n_bits;
 625                                 uint8_t pad[2];
 626                         };
 627
 628                         uint32_t val;
 629                 };
 630         } io;
 631
 632         struct {
 633                 uint8_t header_id[8];
 634                 uint8_t struct_id[8];
 635                 uint8_t n_bytes[8];
 636         } hdr;
 637 };
 638
 639 struct instr_hdr_validity {
 640         uint8_t header_id;
 641 };
 642
 643 struct instr_table {
 644         uint8_t table_id;
 645 };
 646
 647 struct instr_learn {
 648         uint8_t action_id;
 649 };
 650
 651 struct instr_extern_obj {
 652         uint8_t ext_obj_id;
 653         uint8_t func_id;
 654 };
 655
 656 struct instr_extern_func {
 657         uint8_t ext_func_id;
 658 };
 659
 660 struct instr_dst_src {
 661         struct instr_operand dst;
 662         union {
 663                 struct instr_operand src;
 664                 uint64_t src_val;
 665         };
 666 };
 667
 668 struct instr_regarray {
 669         uint8_t regarray_id;
 670         uint8_t pad[3];
 671
 672         union {
 673                 struct instr_operand idx;
 674                 uint32_t idx_val;
 675         };
 676
 677         union {
 678                 struct instr_operand dstsrc;
 679                 uint64_t dstsrc_val;
 680         };
 681 };
 682
 683 struct instr_meter {
 684         uint8_t metarray_id;
 685         uint8_t pad[3];
 686
 687         union {
 688                 struct instr_operand idx;
 689                 uint32_t idx_val;
 690         };
 691
 692         struct instr_operand length;
 693
 694         union {
 695                 struct instr_operand color_in;
 696                 uint32_t color_in_val;
 697         };
 698
 699         struct instr_operand color_out;
 700 };
 701
 702 struct instr_dma {
 703         struct {
 704                 uint8_t header_id[8];
 705                 uint8_t struct_id[8];
 706         } dst;
 707
 708         struct {
 709                 uint8_t offset[8];
 710         } src;
 711
 712         uint16_t n_bytes[8];
 713 };
 714
 715 struct instr_jmp {
 716         struct instruction *ip;
 717
 718         union {
 719                 struct instr_operand a;
 720                 uint8_t header_id;
 721                 uint8_t action_id;
 722         };
 723
 724         union {
 725                 struct instr_operand b;
 726                 uint64_t b_val;
 727         };
 728 };
 729
 730 struct instruction {
 731         enum instruction_type type;
 732         union {
 733                 struct instr_io io;
 734                 struct instr_hdr_validity valid;
 735                 struct instr_dst_src mov;
 736                 struct instr_regarray regarray;
 737                 struct instr_meter meter;
 738                 struct instr_dma dma;
 739                 struct instr_dst_src alu;
 740                 struct instr_table table;
 741                 struct instr_learn learn;
 742                 struct instr_extern_obj ext_obj;
 743                 struct instr_extern_func ext_func;
 744                 struct instr_jmp jmp;
 745         };
 746 };
 747
 748 struct instruction_data {
 749         char label[RTE_SWX_NAME_SIZE];
 750         char jmp_label[RTE_SWX_NAME_SIZE];
 751         uint32_t n_users; /* user = jmp instruction to this instruction. */
 752         int invalid;
 753 };
 754
 755 /*
 756  * Action.
 757  */
 758 struct action {
 759         TAILQ_ENTRY(action) node;
 760         char name[RTE_SWX_NAME_SIZE];
 761         struct struct_type *st;
 762         int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */
 763         struct instruction *instructions;
 764         uint32_t n_instructions;
 765         uint32_t id;
 766 };
 767
 768 TAILQ_HEAD(action_tailq, action);
 769
 770 /*
 771  * Table.
 772  */
 773 struct table_type {
 774         TAILQ_ENTRY(table_type) node;
 775         char name[RTE_SWX_NAME_SIZE];
 776         enum rte_swx_table_match_type match_type;
 777         struct rte_swx_table_ops ops;
 778 };
 779
 780 TAILQ_HEAD(table_type_tailq, table_type);
 781
 782 struct match_field {
 783         enum rte_swx_table_match_type match_type;
 784         struct field *field;
 785 };
 786
 787 struct table {
 788         TAILQ_ENTRY(table) node;
 789         char name[RTE_SWX_NAME_SIZE];
 790         char args[RTE_SWX_NAME_SIZE];
 791         struct table_type *type; /* NULL when n_fields == 0. */
 792
 793         /* Match. */
 794         struct match_field *fields;
 795         uint32_t n_fields;
 796         struct header *header; /* Only valid when n_fields > 0. */
 797
 798         /* Action. */
 799         struct action **actions;
 800         struct action *default_action;
 801         uint8_t *default_action_data;
 802         uint32_t n_actions;
 803         int default_action_is_const;
 804         uint32_t action_data_size_max;
 805
 806         uint32_t size;
 807         uint32_t id;
 808 };
 809
 810 TAILQ_HEAD(table_tailq, table);
 811
 812 struct table_runtime {
 813         rte_swx_table_lookup_t func;
 814         void *mailbox;
 815         uint8_t **key;
 816 };
 817
 818 struct table_statistics {
 819         uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
 820         uint64_t *n_pkts_action;
 821 };
 822
 823 /*
 824  * Selector.
 825  */
 826 struct selector {
 827         TAILQ_ENTRY(selector) node;
 828         char name[RTE_SWX_NAME_SIZE];
 829
 830         struct field *group_id_field;
 831         struct field **selector_fields;
 832         uint32_t n_selector_fields;
 833         struct header *selector_header;
 834         struct field *member_id_field;
 835
 836         uint32_t n_groups_max;
 837         uint32_t n_members_per_group_max;
 838
 839         uint32_t id;
 840 };
 841
 842 TAILQ_HEAD(selector_tailq, selector);
 843
 844 struct selector_runtime {
 845         void *mailbox;
 846         uint8_t **group_id_buffer;
 847         uint8_t **selector_buffer;
 848         uint8_t **member_id_buffer;
 849 };
 850
 851 struct selector_statistics {
 852         uint64_t n_pkts;
 853 };
 854
 855 /*
 856  * Learner table.
 857  */
 858 struct learner {
 859         TAILQ_ENTRY(learner) node;
 860         char name[RTE_SWX_NAME_SIZE];
 861
 862         /* Match. */
 863         struct field **fields;
 864         uint32_t n_fields;
 865         struct header *header;
 866
 867         /* Action. */
 868         struct action **actions;
 869         struct field **action_arg;
 870         struct action *default_action;
 871         uint8_t *default_action_data;
 872         uint32_t n_actions;
 873         int default_action_is_const;
 874         uint32_t action_data_size_max;
 875
 876         uint32_t size;
 877         uint32_t timeout;
 878         uint32_t id;
 879 };
 880
 881 TAILQ_HEAD(learner_tailq, learner);
 882
 883 struct learner_runtime {
 884         void *mailbox;
 885         uint8_t **key;
 886         uint8_t **action_data;
 887 };
 888
 889 struct learner_statistics {
 890         uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
 891         uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */
 892         uint64_t n_pkts_forget;
 893         uint64_t *n_pkts_action;
 894 };
 895
 896 /*
 897  * Register array.
 898  */
 899 struct regarray {
 900         TAILQ_ENTRY(regarray) node;
 901         char name[RTE_SWX_NAME_SIZE];
 902         uint64_t init_val;
 903         uint32_t size;
 904         uint32_t id;
 905 };
 906
 907 TAILQ_HEAD(regarray_tailq, regarray);
 908
 909 struct regarray_runtime {
 910         uint64_t *regarray;
 911         uint32_t size_mask;
 912 };
 913
 914 /*
 915  * Meter array.
 916  */
 917 struct meter_profile {
 918         TAILQ_ENTRY(meter_profile) node;
 919         char name[RTE_SWX_NAME_SIZE];
 920         struct rte_meter_trtcm_params params;
 921         struct rte_meter_trtcm_profile profile;
 922         uint32_t n_users;
 923 };
 924
 925 TAILQ_HEAD(meter_profile_tailq, meter_profile);
 926
 927 struct metarray {
 928         TAILQ_ENTRY(metarray) node;
 929         char name[RTE_SWX_NAME_SIZE];
 930         uint32_t size;
 931         uint32_t id;
 932 };
 933
 934 TAILQ_HEAD(metarray_tailq, metarray);
 935
 936 struct meter {
 937         struct rte_meter_trtcm m;
 938         struct meter_profile *profile;
 939         enum rte_color color_mask;
 940         uint8_t pad[20];
 941
 942         uint64_t n_pkts[RTE_COLORS];
 943         uint64_t n_bytes[RTE_COLORS];
 944 };
 945
 946 struct metarray_runtime {
 947         struct meter *metarray;
 948         uint32_t size_mask;
 949 };
 950
 951 /*
 952  * Pipeline.
 953  */
 954 struct thread {
 955         /* Packet. */
 956         struct rte_swx_pkt pkt;
 957         uint8_t *ptr;
 958
 959         /* Structures. */
 960         uint8_t **structs;
 961
 962         /* Packet headers. */
 963         struct header_runtime *headers; /* Extracted or generated headers. */
 964         struct header_out_runtime *headers_out; /* Emitted headers. */
 965         uint8_t *header_storage;
 966         uint8_t *header_out_storage;
 967         uint64_t valid_headers;
 968         uint32_t n_headers_out;
 969
 970         /* Packet meta-data. */
 971         uint8_t *metadata;
 972
 973         /* Tables. */
 974         struct table_runtime *tables;
 975         struct selector_runtime *selectors;
 976         struct learner_runtime *learners;
 977         struct rte_swx_table_state *table_state;
 978         uint64_t action_id;
 979         int hit; /* 0 = Miss, 1 = Hit. */
 980         uint32_t learner_id;
 981         uint64_t time;
 982
 983         /* Extern objects and functions. */
 984         struct extern_obj_runtime *extern_objs;
 985         struct extern_func_runtime *extern_funcs;
 986
 987         /* Instructions. */
 988         struct instruction *ip;
 989         struct instruction *ret;
 990 };
 991
 992 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 993 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 994 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 995
 996 #define HEADER_VALID(thread, header_id) \
 997         MASK64_BIT_GET((thread)->valid_headers, header_id)
 998
 999 #define ALU(thread, ip, operator)  \
1000 {                                                                              \
1001         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1002         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1003         uint64_t dst64 = *dst64_ptr;                                           \
1004         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1005         uint64_t dst = dst64 & dst64_mask;                                     \
1006                                                                                \
1007         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
1008         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
1009         uint64_t src64 = *src64_ptr;                                           \
1010         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
1011         uint64_t src = src64 & src64_mask;                                     \
1012                                                                                \
1013         uint64_t result = dst operator src;                                    \
1014                                                                                \
1015         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
1016 }
1017
1018 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1019
1020 #define ALU_MH(thread, ip, operator)  \
1021 {                                                                              \
1022         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1023         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1024         uint64_t dst64 = *dst64_ptr;                                           \
1025         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1026         uint64_t dst = dst64 & dst64_mask;                                     \
1027                                                                                \
1028         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
1029         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
1030         uint64_t src64 = *src64_ptr;                                           \
1031         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
1032                                                                                \
1033         uint64_t result = dst operator src;                                    \
1034                                                                                \
1035         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
1036 }
1037
1038 #define ALU_HM(thread, ip, operator)  \
1039 {                                                                              \
1040         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1041         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1042         uint64_t dst64 = *dst64_ptr;                                           \
1043         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1044         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
1045                                                                                \
1046         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
1047         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
1048         uint64_t src64 = *src64_ptr;                                           \
1049         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
1050         uint64_t src = src64 & src64_mask;                                     \
1051                                                                                \
1052         uint64_t result = dst operator src;                                    \
1053         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1054                                                                                \
1055         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1056 }
1057
1058 #define ALU_HM_FAST(thread, ip, operator)  \
1059 {                                                                                 \
1060         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];         \
1061         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];      \
1062         uint64_t dst64 = *dst64_ptr;                                              \
1063         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);          \
1064         uint64_t dst = dst64 & dst64_mask;                                        \
1065                                                                                   \
1066         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];         \
1067         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];      \
1068         uint64_t src64 = *src64_ptr;                                              \
1069         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);          \
1070         uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1071                                                                                   \
1072         uint64_t result = dst operator src;                                       \
1073                                                                                   \
1074         *dst64_ptr = (dst64 & ~dst64_mask) | result;                              \
1075 }
1076
1077 #define ALU_HH(thread, ip, operator)  \
1078 {                                                                              \
1079         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1080         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1081         uint64_t dst64 = *dst64_ptr;                                           \
1082         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1083         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
1084                                                                                \
1085         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
1086         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
1087         uint64_t src64 = *src64_ptr;                                           \
1088         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
1089                                                                                \
1090         uint64_t result = dst operator src;                                    \
1091         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1092                                                                                \
1093         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1094 }
1095
1096 #define ALU_HH_FAST(thread, ip, operator)  \
1097 {                                                                                             \
1098         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];                     \
1099         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];                  \
1100         uint64_t dst64 = *dst64_ptr;                                                          \
1101         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);                      \
1102         uint64_t dst = dst64 & dst64_mask;                                                    \
1103                                                                                               \
1104         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];                     \
1105         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];                  \
1106         uint64_t src64 = *src64_ptr;                                                          \
1107         uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1108                                                                                               \
1109         uint64_t result = dst operator src;                                                   \
1110                                                                                               \
1111         *dst64_ptr = (dst64 & ~dst64_mask) | result;                                          \
1112 }
1113
1114 #else
1115
1116 #define ALU_MH ALU
1117 #define ALU_HM ALU
1118 #define ALU_HM_FAST ALU
1119 #define ALU_HH ALU
1120 #define ALU_HH_FAST ALU
1121
1122 #endif
1123
1124 #define ALU_I(thread, ip, operator)  \
1125 {                                                                              \
1126         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1127         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1128         uint64_t dst64 = *dst64_ptr;                                           \
1129         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1130         uint64_t dst = dst64 & dst64_mask;                                     \
1131                                                                                \
1132         uint64_t src = (ip)->alu.src_val;                                      \
1133                                                                                \
1134         uint64_t result = dst operator src;                                    \
1135                                                                                \
1136         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
1137 }
1138
1139 #define ALU_MI ALU_I
1140
1141 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1142
1143 #define ALU_HI(thread, ip, operator)  \
1144 {                                                                              \
1145         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1146         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1147         uint64_t dst64 = *dst64_ptr;                                           \
1148         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1149         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
1150                                                                                \
1151         uint64_t src = (ip)->alu.src_val;                                      \
1152                                                                                \
1153         uint64_t result = dst operator src;                                    \
1154         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1155                                                                                \
1156         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1157 }
1158
1159 #else
1160
1161 #define ALU_HI ALU_I
1162
1163 #endif
1164
1165 #define MOV(thread, ip)  \
1166 {                                                                              \
1167         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1168         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1169         uint64_t dst64 = *dst64_ptr;                                           \
1170         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1171                                                                                \
1172         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1173         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1174         uint64_t src64 = *src64_ptr;                                           \
1175         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1176         uint64_t src = src64 & src64_mask;                                     \
1177                                                                                \
1178         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1179 }
1180
1181 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1182
1183 #define MOV_MH(thread, ip)  \
1184 {                                                                              \
1185         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1186         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1187         uint64_t dst64 = *dst64_ptr;                                           \
1188         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1189                                                                                \
1190         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1191         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1192         uint64_t src64 = *src64_ptr;                                           \
1193         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
1194                                                                                \
1195         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1196 }
1197
1198 #define MOV_HM(thread, ip)  \
1199 {                                                                              \
1200         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1201         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1202         uint64_t dst64 = *dst64_ptr;                                           \
1203         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1204                                                                                \
1205         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1206         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1207         uint64_t src64 = *src64_ptr;                                           \
1208         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1209         uint64_t src = src64 & src64_mask;                                     \
1210                                                                                \
1211         src = hton64(src) >> (64 - (ip)->mov.dst.n_bits);                      \
1212         *dst64_ptr = (dst64 & ~dst64_mask) | src;                              \
1213 }
1214
1215 #define MOV_HH(thread, ip)  \
1216 {                                                                              \
1217         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1218         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1219         uint64_t dst64 = *dst64_ptr;                                           \
1220         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1221                                                                                \
1222         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1223         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1224         uint64_t src64 = *src64_ptr;                                           \
1225                                                                                \
1226         uint64_t src = src64 << (64 - (ip)->mov.src.n_bits);                   \
1227         src = src >> (64 - (ip)->mov.dst.n_bits);                              \
1228         *dst64_ptr = (dst64 & ~dst64_mask) | src;                              \
1229 }
1230
1231 #else
1232
1233 #define MOV_MH MOV
1234 #define MOV_HM MOV
1235 #define MOV_HH MOV
1236
1237 #endif
1238
1239 #define MOV_I(thread, ip)  \
1240 {                                                                              \
1241         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1242         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1243         uint64_t dst64 = *dst64_ptr;                                           \
1244         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1245                                                                                \
1246         uint64_t src = (ip)->mov.src_val;                                      \
1247                                                                                \
1248         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1249 }
1250
1251 #define JMP_CMP(thread, ip, operator)  \
1252 {                                                                              \
1253         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1254         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1255         uint64_t a64 = *a64_ptr;                                               \
1256         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1257         uint64_t a = a64 & a64_mask;                                           \
1258                                                                                \
1259         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1260         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1261         uint64_t b64 = *b64_ptr;                                               \
1262         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1263         uint64_t b = b64 & b64_mask;                                           \
1264                                                                                \
1265         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1266 }
1267
1268 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1269
1270 #define JMP_CMP_MH(thread, ip, operator)  \
1271 {                                                                              \
1272         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1273         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1274         uint64_t a64 = *a64_ptr;                                               \
1275         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1276         uint64_t a = a64 & a64_mask;                                           \
1277                                                                                \
1278         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1279         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1280         uint64_t b64 = *b64_ptr;                                               \
1281         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1282                                                                                \
1283         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1284 }
1285
1286 #define JMP_CMP_HM(thread, ip, operator)  \
1287 {                                                                              \
1288         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1289         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1290         uint64_t a64 = *a64_ptr;                                               \
1291         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1292                                                                                \
1293         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1294         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1295         uint64_t b64 = *b64_ptr;                                               \
1296         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1297         uint64_t b = b64 & b64_mask;                                           \
1298                                                                                \
1299         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1300 }
1301
1302 #define JMP_CMP_HH(thread, ip, operator)  \
1303 {                                                                              \
1304         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1305         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1306         uint64_t a64 = *a64_ptr;                                               \
1307         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1308                                                                                \
1309         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1310         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1311         uint64_t b64 = *b64_ptr;                                               \
1312         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1313                                                                                \
1314         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1315 }
1316
1317 #define JMP_CMP_HH_FAST(thread, ip, operator)  \
1318 {                                                                              \
1319         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1320         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1321         uint64_t a64 = *a64_ptr;                                               \
1322         uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits);                         \
1323                                                                                \
1324         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1325         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1326         uint64_t b64 = *b64_ptr;                                               \
1327         uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits);                         \
1328                                                                                \
1329         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1330 }
1331
1332 #else
1333
1334 #define JMP_CMP_MH JMP_CMP
1335 #define JMP_CMP_HM JMP_CMP
1336 #define JMP_CMP_HH JMP_CMP
1337 #define JMP_CMP_HH_FAST JMP_CMP
1338
1339 #endif
1340
1341 #define JMP_CMP_I(thread, ip, operator)  \
1342 {                                                                              \
1343         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1344         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1345         uint64_t a64 = *a64_ptr;                                               \
1346         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1347         uint64_t a = a64 & a64_mask;                                           \
1348                                                                                \
1349         uint64_t b = (ip)->jmp.b_val;                                          \
1350                                                                                \
1351         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1352 }
1353
1354 #define JMP_CMP_MI JMP_CMP_I
1355
1356 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1357
1358 #define JMP_CMP_HI(thread, ip, operator)  \
1359 {                                                                              \
1360         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1361         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1362         uint64_t a64 = *a64_ptr;                                               \
1363         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1364                                                                                \
1365         uint64_t b = (ip)->jmp.b_val;                                          \
1366                                                                                \
1367         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1368 }
1369
1370 #else
1371
1372 #define JMP_CMP_HI JMP_CMP_I
1373
1374 #endif
1375
1376 #define METADATA_READ(thread, offset, n_bits)                                  \
1377 ({                                                                             \
1378         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1379         uint64_t m64 = *m64_ptr;                                               \
1380         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1381         (m64 & m64_mask);                                                      \
1382 })
1383
1384 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
1385 {                                                                              \
1386         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1387         uint64_t m64 = *m64_ptr;                                               \
1388         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1389                                                                                \
1390         uint64_t m_new = value;                                                \
1391                                                                                \
1392         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
1393 }
1394
1395 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1396 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1397 #endif
1398
1399 struct rte_swx_pipeline {
1400         struct struct_type_tailq struct_types;
1401         struct port_in_type_tailq port_in_types;
1402         struct port_in_tailq ports_in;
1403         struct port_out_type_tailq port_out_types;
1404         struct port_out_tailq ports_out;
1405         struct extern_type_tailq extern_types;
1406         struct extern_obj_tailq extern_objs;
1407         struct extern_func_tailq extern_funcs;
1408         struct header_tailq headers;
1409         struct struct_type *metadata_st;
1410         uint32_t metadata_struct_id;
1411         struct action_tailq actions;
1412         struct table_type_tailq table_types;
1413         struct table_tailq tables;
1414         struct selector_tailq selectors;
1415         struct learner_tailq learners;
1416         struct regarray_tailq regarrays;
1417         struct meter_profile_tailq meter_profiles;
1418         struct metarray_tailq metarrays;
1419
1420         struct port_in_runtime *in;
1421         struct port_out_runtime *out;
1422         struct instruction **action_instructions;
1423         struct rte_swx_table_state *table_state;
1424         struct table_statistics *table_stats;
1425         struct selector_statistics *selector_stats;
1426         struct learner_statistics *learner_stats;
1427         struct regarray_runtime *regarray_runtime;
1428         struct metarray_runtime *metarray_runtime;
1429         struct instruction *instructions;
1430         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1431
1432         uint32_t n_structs;
1433         uint32_t n_ports_in;
1434         uint32_t n_ports_out;
1435         uint32_t n_extern_objs;
1436         uint32_t n_extern_funcs;
1437         uint32_t n_actions;
1438         uint32_t n_tables;
1439         uint32_t n_selectors;
1440         uint32_t n_learners;
1441         uint32_t n_regarrays;
1442         uint32_t n_metarrays;
1443         uint32_t n_headers;
1444         uint32_t thread_id;
1445         uint32_t port_id;
1446         uint32_t n_instructions;
1447         int build_done;
1448         int numa_node;
1449 };
1450
1451 /*
1452  * Struct.
1453  */
1454 static struct struct_type *
1455 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1456 {
1457         struct struct_type *elem;
1458
1459         TAILQ_FOREACH(elem, &p->struct_types, node)
1460                 if (strcmp(elem->name, name) == 0)
1461                         return elem;
1462
1463         return NULL;
1464 }
1465
1466 static struct field *
1467 struct_type_field_find(struct struct_type *st, const char *name)
1468 {
1469         uint32_t i;
1470
1471         for (i = 0; i < st->n_fields; i++) {
1472                 struct field *f = &st->fields[i];
1473
1474                 if (strcmp(f->name, name) == 0)
1475                         return f;
1476         }
1477
1478         return NULL;
1479 }
1480
1481 int
1482 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1483                                       const char *name,
1484                                       struct rte_swx_field_params *fields,
1485                                       uint32_t n_fields,
1486                                       int last_field_has_variable_size)
1487 {
1488         struct struct_type *st;
1489         uint32_t i;
1490
1491         CHECK(p, EINVAL);
1492         CHECK_NAME(name, EINVAL);
1493         CHECK(fields, EINVAL);
1494         CHECK(n_fields, EINVAL);
1495
1496         for (i = 0; i < n_fields; i++) {
1497                 struct rte_swx_field_params *f = &fields[i];
1498                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
1499                 uint32_t j;
1500
1501                 CHECK_NAME(f->name, EINVAL);
1502                 CHECK(f->n_bits, EINVAL);
1503                 CHECK((f->n_bits <= 64) || var_size, EINVAL);
1504                 CHECK((f->n_bits & 7) == 0, EINVAL);
1505
1506                 for (j = 0; j < i; j++) {
1507                         struct rte_swx_field_params *f_prev = &fields[j];
1508
1509                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
1510                 }
1511         }
1512
1513         CHECK(!struct_type_find(p, name), EEXIST);
1514
1515         /* Node allocation. */
1516         st = calloc(1, sizeof(struct struct_type));
1517         CHECK(st, ENOMEM);
1518
1519         st->fields = calloc(n_fields, sizeof(struct field));
1520         if (!st->fields) {
1521                 free(st);
1522                 CHECK(0, ENOMEM);
1523         }
1524
1525         /* Node initialization. */
1526         strcpy(st->name, name);
1527         for (i = 0; i < n_fields; i++) {
1528                 struct field *dst = &st->fields[i];
1529                 struct rte_swx_field_params *src = &fields[i];
1530                 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
1531
1532                 strcpy(dst->name, src->name);
1533                 dst->n_bits = src->n_bits;
1534                 dst->offset = st->n_bits;
1535                 dst->var_size = var_size;
1536
1537                 st->n_bits += src->n_bits;
1538                 st->n_bits_min += var_size ? 0 : src->n_bits;
1539         }
1540         st->n_fields = n_fields;
1541         st->var_size = last_field_has_variable_size;
1542
1543         /* Node add to tailq. */
1544         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1545
1546         return 0;
1547 }
1548
1549 static int
1550 struct_build(struct rte_swx_pipeline *p)
1551 {
1552         uint32_t i;
1553
1554         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1555                 struct thread *t = &p->threads[i];
1556
1557                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1558                 CHECK(t->structs, ENOMEM);
1559         }
1560
1561         return 0;
1562 }
1563
1564 static void
1565 struct_build_free(struct rte_swx_pipeline *p)
1566 {
1567         uint32_t i;
1568
1569         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1570                 struct thread *t = &p->threads[i];
1571
1572                 free(t->structs);
1573                 t->structs = NULL;
1574         }
1575 }
1576
1577 static void
1578 struct_free(struct rte_swx_pipeline *p)
1579 {
1580         struct_build_free(p);
1581
1582         /* Struct types. */
1583         for ( ; ; ) {
1584                 struct struct_type *elem;
1585
1586                 elem = TAILQ_FIRST(&p->struct_types);
1587                 if (!elem)
1588                         break;
1589
1590                 TAILQ_REMOVE(&p->struct_types, elem, node);
1591                 free(elem->fields);
1592                 free(elem);
1593         }
1594 }
1595
1596 /*
1597  * Input port.
1598  */
1599 static struct port_in_type *
1600 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1601 {
1602         struct port_in_type *elem;
1603
1604         if (!name)
1605                 return NULL;
1606
1607         TAILQ_FOREACH(elem, &p->port_in_types, node)
1608                 if (strcmp(elem->name, name) == 0)
1609                         return elem;
1610
1611         return NULL;
1612 }
1613
1614 int
1615 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1616                                        const char *name,
1617                                        struct rte_swx_port_in_ops *ops)
1618 {
1619         struct port_in_type *elem;
1620
1621         CHECK(p, EINVAL);
1622         CHECK_NAME(name, EINVAL);
1623         CHECK(ops, EINVAL);
1624         CHECK(ops->create, EINVAL);
1625         CHECK(ops->free, EINVAL);
1626         CHECK(ops->pkt_rx, EINVAL);
1627         CHECK(ops->stats_read, EINVAL);
1628
1629         CHECK(!port_in_type_find(p, name), EEXIST);
1630
1631         /* Node allocation. */
1632         elem = calloc(1, sizeof(struct port_in_type));
1633         CHECK(elem, ENOMEM);
1634
1635         /* Node initialization. */
1636         strcpy(elem->name, name);
1637         memcpy(&elem->ops, ops, sizeof(*ops));
1638
1639         /* Node add to tailq. */
1640         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1641
1642         return 0;
1643 }
1644
1645 static struct port_in *
1646 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1647 {
1648         struct port_in *port;
1649
1650         TAILQ_FOREACH(port, &p->ports_in, node)
1651                 if (port->id == port_id)
1652                         return port;
1653
1654         return NULL;
1655 }
1656
1657 int
1658 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1659                                 uint32_t port_id,
1660                                 const char *port_type_name,
1661                                 void *args)
1662 {
1663         struct port_in_type *type = NULL;
1664         struct port_in *port = NULL;
1665         void *obj = NULL;
1666
1667         CHECK(p, EINVAL);
1668
1669         CHECK(!port_in_find(p, port_id), EINVAL);
1670
1671         CHECK_NAME(port_type_name, EINVAL);
1672         type = port_in_type_find(p, port_type_name);
1673         CHECK(type, EINVAL);
1674
1675         obj = type->ops.create(args);
1676         CHECK(obj, ENODEV);
1677
1678         /* Node allocation. */
1679         port = calloc(1, sizeof(struct port_in));
1680         CHECK(port, ENOMEM);
1681
1682         /* Node initialization. */
1683         port->type = type;
1684         port->obj = obj;
1685         port->id = port_id;
1686
1687         /* Node add to tailq. */
1688         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1689         if (p->n_ports_in < port_id + 1)
1690                 p->n_ports_in = port_id + 1;
1691
1692         return 0;
1693 }
1694
1695 static int
1696 port_in_build(struct rte_swx_pipeline *p)
1697 {
1698         struct port_in *port;
1699         uint32_t i;
1700
1701         CHECK(p->n_ports_in, EINVAL);
1702         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1703
1704         for (i = 0; i < p->n_ports_in; i++)
1705                 CHECK(port_in_find(p, i), EINVAL);
1706
1707         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1708         CHECK(p->in, ENOMEM);
1709
1710         TAILQ_FOREACH(port, &p->ports_in, node) {
1711                 struct port_in_runtime *in = &p->in[port->id];
1712
1713                 in->pkt_rx = port->type->ops.pkt_rx;
1714                 in->obj = port->obj;
1715         }
1716
1717         return 0;
1718 }
1719
1720 static void
1721 port_in_build_free(struct rte_swx_pipeline *p)
1722 {
1723         free(p->in);
1724         p->in = NULL;
1725 }
1726
1727 static void
1728 port_in_free(struct rte_swx_pipeline *p)
1729 {
1730         port_in_build_free(p);
1731
1732         /* Input ports. */
1733         for ( ; ; ) {
1734                 struct port_in *port;
1735
1736                 port = TAILQ_FIRST(&p->ports_in);
1737                 if (!port)
1738                         break;
1739
1740                 TAILQ_REMOVE(&p->ports_in, port, node);
1741                 port->type->ops.free(port->obj);
1742                 free(port);
1743         }
1744
1745         /* Input port types. */
1746         for ( ; ; ) {
1747                 struct port_in_type *elem;
1748
1749                 elem = TAILQ_FIRST(&p->port_in_types);
1750                 if (!elem)
1751                         break;
1752
1753                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1754                 free(elem);
1755         }
1756 }
1757
1758 /*
1759  * Output port.
1760  */
1761 static struct port_out_type *
1762 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1763 {
1764         struct port_out_type *elem;
1765
1766         if (!name)
1767                 return NULL;
1768
1769         TAILQ_FOREACH(elem, &p->port_out_types, node)
1770                 if (!strcmp(elem->name, name))
1771                         return elem;
1772
1773         return NULL;
1774 }
1775
1776 int
1777 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1778                                         const char *name,
1779                                         struct rte_swx_port_out_ops *ops)
1780 {
1781         struct port_out_type *elem;
1782
1783         CHECK(p, EINVAL);
1784         CHECK_NAME(name, EINVAL);
1785         CHECK(ops, EINVAL);
1786         CHECK(ops->create, EINVAL);
1787         CHECK(ops->free, EINVAL);
1788         CHECK(ops->pkt_tx, EINVAL);
1789         CHECK(ops->stats_read, EINVAL);
1790
1791         CHECK(!port_out_type_find(p, name), EEXIST);
1792
1793         /* Node allocation. */
1794         elem = calloc(1, sizeof(struct port_out_type));
1795         CHECK(elem, ENOMEM);
1796
1797         /* Node initialization. */
1798         strcpy(elem->name, name);
1799         memcpy(&elem->ops, ops, sizeof(*ops));
1800
1801         /* Node add to tailq. */
1802         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1803
1804         return 0;
1805 }
1806
1807 static struct port_out *
1808 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1809 {
1810         struct port_out *port;
1811
1812         TAILQ_FOREACH(port, &p->ports_out, node)
1813                 if (port->id == port_id)
1814                         return port;
1815
1816         return NULL;
1817 }
1818
1819 int
1820 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1821                                  uint32_t port_id,
1822                                  const char *port_type_name,
1823                                  void *args)
1824 {
1825         struct port_out_type *type = NULL;
1826         struct port_out *port = NULL;
1827         void *obj = NULL;
1828
1829         CHECK(p, EINVAL);
1830
1831         CHECK(!port_out_find(p, port_id), EINVAL);
1832
1833         CHECK_NAME(port_type_name, EINVAL);
1834         type = port_out_type_find(p, port_type_name);
1835         CHECK(type, EINVAL);
1836
1837         obj = type->ops.create(args);
1838         CHECK(obj, ENODEV);
1839
1840         /* Node allocation. */
1841         port = calloc(1, sizeof(struct port_out));
1842         CHECK(port, ENOMEM);
1843
1844         /* Node initialization. */
1845         port->type = type;
1846         port->obj = obj;
1847         port->id = port_id;
1848
1849         /* Node add to tailq. */
1850         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1851         if (p->n_ports_out < port_id + 1)
1852                 p->n_ports_out = port_id + 1;
1853
1854         return 0;
1855 }
1856
1857 static int
1858 port_out_build(struct rte_swx_pipeline *p)
1859 {
1860         struct port_out *port;
1861         uint32_t i;
1862
1863         CHECK(p->n_ports_out, EINVAL);
1864
1865         for (i = 0; i < p->n_ports_out; i++)
1866                 CHECK(port_out_find(p, i), EINVAL);
1867
1868         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1869         CHECK(p->out, ENOMEM);
1870
1871         TAILQ_FOREACH(port, &p->ports_out, node) {
1872                 struct port_out_runtime *out = &p->out[port->id];
1873
1874                 out->pkt_tx = port->type->ops.pkt_tx;
1875                 out->flush = port->type->ops.flush;
1876                 out->obj = port->obj;
1877         }
1878
1879         return 0;
1880 }
1881
1882 static void
1883 port_out_build_free(struct rte_swx_pipeline *p)
1884 {
1885         free(p->out);
1886         p->out = NULL;
1887 }
1888
1889 static void
1890 port_out_free(struct rte_swx_pipeline *p)
1891 {
1892         port_out_build_free(p);
1893
1894         /* Output ports. */
1895         for ( ; ; ) {
1896                 struct port_out *port;
1897
1898                 port = TAILQ_FIRST(&p->ports_out);
1899                 if (!port)
1900                         break;
1901
1902                 TAILQ_REMOVE(&p->ports_out, port, node);
1903                 port->type->ops.free(port->obj);
1904                 free(port);
1905         }
1906
1907         /* Output port types. */
1908         for ( ; ; ) {
1909                 struct port_out_type *elem;
1910
1911                 elem = TAILQ_FIRST(&p->port_out_types);
1912                 if (!elem)
1913                         break;
1914
1915                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1916                 free(elem);
1917         }
1918 }
1919
1920 /*
1921  * Extern object.
1922  */
1923 static struct extern_type *
1924 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1925 {
1926         struct extern_type *elem;
1927
1928         TAILQ_FOREACH(elem, &p->extern_types, node)
1929                 if (strcmp(elem->name, name) == 0)
1930                         return elem;
1931
1932         return NULL;
1933 }
1934
1935 static struct extern_type_member_func *
1936 extern_type_member_func_find(struct extern_type *type, const char *name)
1937 {
1938         struct extern_type_member_func *elem;
1939
1940         TAILQ_FOREACH(elem, &type->funcs, node)
1941                 if (strcmp(elem->name, name) == 0)
1942                         return elem;
1943
1944         return NULL;
1945 }
1946
1947 static struct extern_obj *
1948 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1949 {
1950         struct extern_obj *elem;
1951
1952         TAILQ_FOREACH(elem, &p->extern_objs, node)
1953                 if (strcmp(elem->name, name) == 0)
1954                         return elem;
1955
1956         return NULL;
1957 }
1958
1959 static struct extern_type_member_func *
1960 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1961                              const char *name,
1962                              struct extern_obj **obj)
1963 {
1964         struct extern_obj *object;
1965         struct extern_type_member_func *func;
1966         char *object_name, *func_name;
1967
1968         if (name[0] != 'e' || name[1] != '.')
1969                 return NULL;
1970
1971         object_name = strdup(&name[2]);
1972         if (!object_name)
1973                 return NULL;
1974
1975         func_name = strchr(object_name, '.');
1976         if (!func_name) {
1977                 free(object_name);
1978                 return NULL;
1979         }
1980
1981         *func_name = 0;
1982         func_name++;
1983
1984         object = extern_obj_find(p, object_name);
1985         if (!object) {
1986                 free(object_name);
1987                 return NULL;
1988         }
1989
1990         func = extern_type_member_func_find(object->type, func_name);
1991         if (!func) {
1992                 free(object_name);
1993                 return NULL;
1994         }
1995
1996         if (obj)
1997                 *obj = object;
1998
1999         free(object_name);
2000         return func;
2001 }
2002
2003 static struct field *
2004 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
2005                                const char *name,
2006                                struct extern_obj **object)
2007 {
2008         struct extern_obj *obj;
2009         struct field *f;
2010         char *obj_name, *field_name;
2011
2012         if ((name[0] != 'e') || (name[1] != '.'))
2013                 return NULL;
2014
2015         obj_name = strdup(&name[2]);
2016         if (!obj_name)
2017                 return NULL;
2018
2019         field_name = strchr(obj_name, '.');
2020         if (!field_name) {
2021                 free(obj_name);
2022                 return NULL;
2023         }
2024
2025         *field_name = 0;
2026         field_name++;
2027
2028         obj = extern_obj_find(p, obj_name);
2029         if (!obj) {
2030                 free(obj_name);
2031                 return NULL;
2032         }
2033
2034         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
2035         if (!f) {
2036                 free(obj_name);
2037                 return NULL;
2038         }
2039
2040         if (object)
2041                 *object = obj;
2042
2043         free(obj_name);
2044         return f;
2045 }
2046
2047 int
2048 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
2049         const char *name,
2050         const char *mailbox_struct_type_name,
2051         rte_swx_extern_type_constructor_t constructor,
2052         rte_swx_extern_type_destructor_t destructor)
2053 {
2054         struct extern_type *elem;
2055         struct struct_type *mailbox_struct_type;
2056
2057         CHECK(p, EINVAL);
2058
2059         CHECK_NAME(name, EINVAL);
2060         CHECK(!extern_type_find(p, name), EEXIST);
2061
2062         CHECK_NAME(mailbox_struct_type_name, EINVAL);
2063         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2064         CHECK(mailbox_struct_type, EINVAL);
2065         CHECK(!mailbox_struct_type->var_size, EINVAL);
2066
2067         CHECK(constructor, EINVAL);
2068         CHECK(destructor, EINVAL);
2069
2070         /* Node allocation. */
2071         elem = calloc(1, sizeof(struct extern_type));
2072         CHECK(elem, ENOMEM);
2073
2074         /* Node initialization. */
2075         strcpy(elem->name, name);
2076         elem->mailbox_struct_type = mailbox_struct_type;
2077         elem->constructor = constructor;
2078         elem->destructor = destructor;
2079         TAILQ_INIT(&elem->funcs);
2080
2081         /* Node add to tailq. */
2082         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
2083
2084         return 0;
2085 }
2086
2087 int
2088 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
2089         const char *extern_type_name,
2090         const char *name,
2091         rte_swx_extern_type_member_func_t member_func)
2092 {
2093         struct extern_type *type;
2094         struct extern_type_member_func *type_member;
2095
2096         CHECK(p, EINVAL);
2097
2098         CHECK_NAME(extern_type_name, EINVAL);
2099         type = extern_type_find(p, extern_type_name);
2100         CHECK(type, EINVAL);
2101         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
2102
2103         CHECK_NAME(name, EINVAL);
2104         CHECK(!extern_type_member_func_find(type, name), EEXIST);
2105
2106         CHECK(member_func, EINVAL);
2107
2108         /* Node allocation. */
2109         type_member = calloc(1, sizeof(struct extern_type_member_func));
2110         CHECK(type_member, ENOMEM);
2111
2112         /* Node initialization. */
2113         strcpy(type_member->name, name);
2114         type_member->func = member_func;
2115         type_member->id = type->n_funcs;
2116
2117         /* Node add to tailq. */
2118         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
2119         type->n_funcs++;
2120
2121         return 0;
2122 }
2123
2124 int
2125 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
2126                                       const char *extern_type_name,
2127                                       const char *name,
2128                                       const char *args)
2129 {
2130         struct extern_type *type;
2131         struct extern_obj *obj;
2132         void *obj_handle;
2133
2134         CHECK(p, EINVAL);
2135
2136         CHECK_NAME(extern_type_name, EINVAL);
2137         type = extern_type_find(p, extern_type_name);
2138         CHECK(type, EINVAL);
2139
2140         CHECK_NAME(name, EINVAL);
2141         CHECK(!extern_obj_find(p, name), EEXIST);
2142
2143         /* Node allocation. */
2144         obj = calloc(1, sizeof(struct extern_obj));
2145         CHECK(obj, ENOMEM);
2146
2147         /* Object construction. */
2148         obj_handle = type->constructor(args);
2149         if (!obj_handle) {
2150                 free(obj);
2151                 CHECK(0, ENODEV);
2152         }
2153
2154         /* Node initialization. */
2155         strcpy(obj->name, name);
2156         obj->type = type;
2157         obj->obj = obj_handle;
2158         obj->struct_id = p->n_structs;
2159         obj->id = p->n_extern_objs;
2160
2161         /* Node add to tailq. */
2162         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
2163         p->n_extern_objs++;
2164         p->n_structs++;
2165
2166         return 0;
2167 }
2168
2169 static int
2170 extern_obj_build(struct rte_swx_pipeline *p)
2171 {
2172         uint32_t i;
2173
2174         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2175                 struct thread *t = &p->threads[i];
2176                 struct extern_obj *obj;
2177
2178                 t->extern_objs = calloc(p->n_extern_objs,
2179                                         sizeof(struct extern_obj_runtime));
2180                 CHECK(t->extern_objs, ENOMEM);
2181
2182                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
2183                         struct extern_obj_runtime *r =
2184                                 &t->extern_objs[obj->id];
2185                         struct extern_type_member_func *func;
2186                         uint32_t mailbox_size =
2187                                 obj->type->mailbox_struct_type->n_bits / 8;
2188
2189                         r->obj = obj->obj;
2190
2191                         r->mailbox = calloc(1, mailbox_size);
2192                         CHECK(r->mailbox, ENOMEM);
2193
2194                         TAILQ_FOREACH(func, &obj->type->funcs, node)
2195                                 r->funcs[func->id] = func->func;
2196
2197                         t->structs[obj->struct_id] = r->mailbox;
2198                 }
2199         }
2200
2201         return 0;
2202 }
2203
2204 static void
2205 extern_obj_build_free(struct rte_swx_pipeline *p)
2206 {
2207         uint32_t i;
2208
2209         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2210                 struct thread *t = &p->threads[i];
2211                 uint32_t j;
2212
2213                 if (!t->extern_objs)
2214                         continue;
2215
2216                 for (j = 0; j < p->n_extern_objs; j++) {
2217                         struct extern_obj_runtime *r = &t->extern_objs[j];
2218
2219                         free(r->mailbox);
2220                 }
2221
2222                 free(t->extern_objs);
2223                 t->extern_objs = NULL;
2224         }
2225 }
2226
2227 static void
2228 extern_obj_free(struct rte_swx_pipeline *p)
2229 {
2230         extern_obj_build_free(p);
2231
2232         /* Extern objects. */
2233         for ( ; ; ) {
2234                 struct extern_obj *elem;
2235
2236                 elem = TAILQ_FIRST(&p->extern_objs);
2237                 if (!elem)
2238                         break;
2239
2240                 TAILQ_REMOVE(&p->extern_objs, elem, node);
2241                 if (elem->obj)
2242                         elem->type->destructor(elem->obj);
2243                 free(elem);
2244         }
2245
2246         /* Extern types. */
2247         for ( ; ; ) {
2248                 struct extern_type *elem;
2249
2250                 elem = TAILQ_FIRST(&p->extern_types);
2251                 if (!elem)
2252                         break;
2253
2254                 TAILQ_REMOVE(&p->extern_types, elem, node);
2255
2256                 for ( ; ; ) {
2257                         struct extern_type_member_func *func;
2258
2259                         func = TAILQ_FIRST(&elem->funcs);
2260                         if (!func)
2261                                 break;
2262
2263                         TAILQ_REMOVE(&elem->funcs, func, node);
2264                         free(func);
2265                 }
2266
2267                 free(elem);
2268         }
2269 }
2270
2271 /*
2272  * Extern function.
2273  */
2274 static struct extern_func *
2275 extern_func_find(struct rte_swx_pipeline *p, const char *name)
2276 {
2277         struct extern_func *elem;
2278
2279         TAILQ_FOREACH(elem, &p->extern_funcs, node)
2280                 if (strcmp(elem->name, name) == 0)
2281                         return elem;
2282
2283         return NULL;
2284 }
2285
2286 static struct extern_func *
2287 extern_func_parse(struct rte_swx_pipeline *p,
2288                   const char *name)
2289 {
2290         if (name[0] != 'f' || name[1] != '.')
2291                 return NULL;
2292
2293         return extern_func_find(p, &name[2]);
2294 }
2295
2296 static struct field *
2297 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
2298                                 const char *name,
2299                                 struct extern_func **function)
2300 {
2301         struct extern_func *func;
2302         struct field *f;
2303         char *func_name, *field_name;
2304
2305         if ((name[0] != 'f') || (name[1] != '.'))
2306                 return NULL;
2307
2308         func_name = strdup(&name[2]);
2309         if (!func_name)
2310                 return NULL;
2311
2312         field_name = strchr(func_name, '.');
2313         if (!field_name) {
2314                 free(func_name);
2315                 return NULL;
2316         }
2317
2318         *field_name = 0;
2319         field_name++;
2320
2321         func = extern_func_find(p, func_name);
2322         if (!func) {
2323                 free(func_name);
2324                 return NULL;
2325         }
2326
2327         f = struct_type_field_find(func->mailbox_struct_type, field_name);
2328         if (!f) {
2329                 free(func_name);
2330                 return NULL;
2331         }
2332
2333         if (function)
2334                 *function = func;
2335
2336         free(func_name);
2337         return f;
2338 }
2339
2340 int
2341 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
2342                                       const char *name,
2343                                       const char *mailbox_struct_type_name,
2344                                       rte_swx_extern_func_t func)
2345 {
2346         struct extern_func *f;
2347         struct struct_type *mailbox_struct_type;
2348
2349         CHECK(p, EINVAL);
2350
2351         CHECK_NAME(name, EINVAL);
2352         CHECK(!extern_func_find(p, name), EEXIST);
2353
2354         CHECK_NAME(mailbox_struct_type_name, EINVAL);
2355         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2356         CHECK(mailbox_struct_type, EINVAL);
2357         CHECK(!mailbox_struct_type->var_size, EINVAL);
2358
2359         CHECK(func, EINVAL);
2360
2361         /* Node allocation. */
2362         f = calloc(1, sizeof(struct extern_func));
2363         CHECK(func, ENOMEM);
2364
2365         /* Node initialization. */
2366         strcpy(f->name, name);
2367         f->mailbox_struct_type = mailbox_struct_type;
2368         f->func = func;
2369         f->struct_id = p->n_structs;
2370         f->id = p->n_extern_funcs;
2371
2372         /* Node add to tailq. */
2373         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
2374         p->n_extern_funcs++;
2375         p->n_structs++;
2376
2377         return 0;
2378 }
2379
2380 static int
2381 extern_func_build(struct rte_swx_pipeline *p)
2382 {
2383         uint32_t i;
2384
2385         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2386                 struct thread *t = &p->threads[i];
2387                 struct extern_func *func;
2388
2389                 /* Memory allocation. */
2390                 t->extern_funcs = calloc(p->n_extern_funcs,
2391                                          sizeof(struct extern_func_runtime));
2392                 CHECK(t->extern_funcs, ENOMEM);
2393
2394                 /* Extern function. */
2395                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
2396                         struct extern_func_runtime *r =
2397                                 &t->extern_funcs[func->id];
2398                         uint32_t mailbox_size =
2399                                 func->mailbox_struct_type->n_bits / 8;
2400
2401                         r->func = func->func;
2402
2403                         r->mailbox = calloc(1, mailbox_size);
2404                         CHECK(r->mailbox, ENOMEM);
2405
2406                         t->structs[func->struct_id] = r->mailbox;
2407                 }
2408         }
2409
2410         return 0;
2411 }
2412
2413 static void
2414 extern_func_build_free(struct rte_swx_pipeline *p)
2415 {
2416         uint32_t i;
2417
2418         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2419                 struct thread *t = &p->threads[i];
2420                 uint32_t j;
2421
2422                 if (!t->extern_funcs)
2423                         continue;
2424
2425                 for (j = 0; j < p->n_extern_funcs; j++) {
2426                         struct extern_func_runtime *r = &t->extern_funcs[j];
2427
2428                         free(r->mailbox);
2429                 }
2430
2431                 free(t->extern_funcs);
2432                 t->extern_funcs = NULL;
2433         }
2434 }
2435
2436 static void
2437 extern_func_free(struct rte_swx_pipeline *p)
2438 {
2439         extern_func_build_free(p);
2440
2441         for ( ; ; ) {
2442                 struct extern_func *elem;
2443
2444                 elem = TAILQ_FIRST(&p->extern_funcs);
2445                 if (!elem)
2446                         break;
2447
2448                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
2449                 free(elem);
2450         }
2451 }
2452
2453 /*
2454  * Header.
2455  */
2456 static struct header *
2457 header_find(struct rte_swx_pipeline *p, const char *name)
2458 {
2459         struct header *elem;
2460
2461         TAILQ_FOREACH(elem, &p->headers, node)
2462                 if (strcmp(elem->name, name) == 0)
2463                         return elem;
2464
2465         return NULL;
2466 }
2467
2468 static struct header *
2469 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
2470 {
2471         struct header *elem;
2472
2473         TAILQ_FOREACH(elem, &p->headers, node)
2474                 if (elem->struct_id == struct_id)
2475                         return elem;
2476
2477         return NULL;
2478 }
2479
2480 static struct header *
2481 header_parse(struct rte_swx_pipeline *p,
2482              const char *name)
2483 {
2484         if (name[0] != 'h' || name[1] != '.')
2485                 return NULL;
2486
2487         return header_find(p, &name[2]);
2488 }
2489
2490 static struct field *
2491 header_field_parse(struct rte_swx_pipeline *p,
2492                    const char *name,
2493                    struct header **header)
2494 {
2495         struct header *h;
2496         struct field *f;
2497         char *header_name, *field_name;
2498
2499         if ((name[0] != 'h') || (name[1] != '.'))
2500                 return NULL;
2501
2502         header_name = strdup(&name[2]);
2503         if (!header_name)
2504                 return NULL;
2505
2506         field_name = strchr(header_name, '.');
2507         if (!field_name) {
2508                 free(header_name);
2509                 return NULL;
2510         }
2511
2512         *field_name = 0;
2513         field_name++;
2514
2515         h = header_find(p, header_name);
2516         if (!h) {
2517                 free(header_name);
2518                 return NULL;
2519         }
2520
2521         f = struct_type_field_find(h->st, field_name);
2522         if (!f) {
2523                 free(header_name);
2524                 return NULL;
2525         }
2526
2527         if (header)
2528                 *header = h;
2529
2530         free(header_name);
2531         return f;
2532 }
2533
2534 int
2535 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2536                                         const char *name,
2537                                         const char *struct_type_name)
2538 {
2539         struct struct_type *st;
2540         struct header *h;
2541         size_t n_headers_max;
2542
2543         CHECK(p, EINVAL);
2544         CHECK_NAME(name, EINVAL);
2545         CHECK_NAME(struct_type_name, EINVAL);
2546
2547         CHECK(!header_find(p, name), EEXIST);
2548
2549         st = struct_type_find(p, struct_type_name);
2550         CHECK(st, EINVAL);
2551
2552         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2553         CHECK(p->n_headers < n_headers_max, ENOSPC);
2554
2555         /* Node allocation. */
2556         h = calloc(1, sizeof(struct header));
2557         CHECK(h, ENOMEM);
2558
2559         /* Node initialization. */
2560         strcpy(h->name, name);
2561         h->st = st;
2562         h->struct_id = p->n_structs;
2563         h->id = p->n_headers;
2564
2565         /* Node add to tailq. */
2566         TAILQ_INSERT_TAIL(&p->headers, h, node);
2567         p->n_headers++;
2568         p->n_structs++;
2569
2570         return 0;
2571 }
2572
2573 static int
2574 header_build(struct rte_swx_pipeline *p)
2575 {
2576         struct header *h;
2577         uint32_t n_bytes = 0, i;
2578
2579         TAILQ_FOREACH(h, &p->headers, node) {
2580                 n_bytes += h->st->n_bits / 8;
2581         }
2582
2583         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2584                 struct thread *t = &p->threads[i];
2585                 uint32_t offset = 0;
2586
2587                 t->headers = calloc(p->n_headers,
2588                                     sizeof(struct header_runtime));
2589                 CHECK(t->headers, ENOMEM);
2590
2591                 t->headers_out = calloc(p->n_headers,
2592                                         sizeof(struct header_out_runtime));
2593                 CHECK(t->headers_out, ENOMEM);
2594
2595                 t->header_storage = calloc(1, n_bytes);
2596                 CHECK(t->header_storage, ENOMEM);
2597
2598                 t->header_out_storage = calloc(1, n_bytes);
2599                 CHECK(t->header_out_storage, ENOMEM);
2600
2601                 TAILQ_FOREACH(h, &p->headers, node) {
2602                         uint8_t *header_storage;
2603                         uint32_t n_bytes =  h->st->n_bits / 8;
2604
2605                         header_storage = &t->header_storage[offset];
2606                         offset += n_bytes;
2607
2608                         t->headers[h->id].ptr0 = header_storage;
2609                         t->headers[h->id].n_bytes = n_bytes;
2610
2611                         t->structs[h->struct_id] = header_storage;
2612                 }
2613         }
2614
2615         return 0;
2616 }
2617
2618 static void
2619 header_build_free(struct rte_swx_pipeline *p)
2620 {
2621         uint32_t i;
2622
2623         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2624                 struct thread *t = &p->threads[i];
2625
2626                 free(t->headers_out);
2627                 t->headers_out = NULL;
2628
2629                 free(t->headers);
2630                 t->headers = NULL;
2631
2632                 free(t->header_out_storage);
2633                 t->header_out_storage = NULL;
2634
2635                 free(t->header_storage);
2636                 t->header_storage = NULL;
2637         }
2638 }
2639
2640 static void
2641 header_free(struct rte_swx_pipeline *p)
2642 {
2643         header_build_free(p);
2644
2645         for ( ; ; ) {
2646                 struct header *elem;
2647
2648                 elem = TAILQ_FIRST(&p->headers);
2649                 if (!elem)
2650                         break;
2651
2652                 TAILQ_REMOVE(&p->headers, elem, node);
2653                 free(elem);
2654         }
2655 }
2656
2657 /*
2658  * Meta-data.
2659  */
2660 static struct field *
2661 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2662 {
2663         if (!p->metadata_st)
2664                 return NULL;
2665
2666         if (name[0] != 'm' || name[1] != '.')
2667                 return NULL;
2668
2669         return struct_type_field_find(p->metadata_st, &name[2]);
2670 }
2671
2672 int
2673 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2674                                           const char *struct_type_name)
2675 {
2676         struct struct_type *st = NULL;
2677
2678         CHECK(p, EINVAL);
2679
2680         CHECK_NAME(struct_type_name, EINVAL);
2681         st  = struct_type_find(p, struct_type_name);
2682         CHECK(st, EINVAL);
2683         CHECK(!st->var_size, EINVAL);
2684         CHECK(!p->metadata_st, EINVAL);
2685
2686         p->metadata_st = st;
2687         p->metadata_struct_id = p->n_structs;
2688
2689         p->n_structs++;
2690
2691         return 0;
2692 }
2693
2694 static int
2695 metadata_build(struct rte_swx_pipeline *p)
2696 {
2697         uint32_t n_bytes = p->metadata_st->n_bits / 8;
2698         uint32_t i;
2699
2700         /* Thread-level initialization. */
2701         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2702                 struct thread *t = &p->threads[i];
2703                 uint8_t *metadata;
2704
2705                 metadata = calloc(1, n_bytes);
2706                 CHECK(metadata, ENOMEM);
2707
2708                 t->metadata = metadata;
2709                 t->structs[p->metadata_struct_id] = metadata;
2710         }
2711
2712         return 0;
2713 }
2714
2715 static void
2716 metadata_build_free(struct rte_swx_pipeline *p)
2717 {
2718         uint32_t i;
2719
2720         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2721                 struct thread *t = &p->threads[i];
2722
2723                 free(t->metadata);
2724                 t->metadata = NULL;
2725         }
2726 }
2727
2728 static void
2729 metadata_free(struct rte_swx_pipeline *p)
2730 {
2731         metadata_build_free(p);
2732 }
2733
2734 /*
2735  * Instruction.
2736  */
2737 static int
2738 instruction_is_tx(enum instruction_type type)
2739 {
2740         switch (type) {
2741         case INSTR_TX:
2742         case INSTR_TX_I:
2743                 return 1;
2744
2745         default:
2746                 return 0;
2747         }
2748 }
2749
2750 static int
2751 instruction_is_jmp(struct instruction *instr)
2752 {
2753         switch (instr->type) {
2754         case INSTR_JMP:
2755         case INSTR_JMP_VALID:
2756         case INSTR_JMP_INVALID:
2757         case INSTR_JMP_HIT:
2758         case INSTR_JMP_MISS:
2759         case INSTR_JMP_ACTION_HIT:
2760         case INSTR_JMP_ACTION_MISS:
2761         case INSTR_JMP_EQ:
2762         case INSTR_JMP_EQ_MH:
2763         case INSTR_JMP_EQ_HM:
2764         case INSTR_JMP_EQ_HH:
2765         case INSTR_JMP_EQ_I:
2766         case INSTR_JMP_NEQ:
2767         case INSTR_JMP_NEQ_MH:
2768         case INSTR_JMP_NEQ_HM:
2769         case INSTR_JMP_NEQ_HH:
2770         case INSTR_JMP_NEQ_I:
2771         case INSTR_JMP_LT:
2772         case INSTR_JMP_LT_MH:
2773         case INSTR_JMP_LT_HM:
2774         case INSTR_JMP_LT_HH:
2775         case INSTR_JMP_LT_MI:
2776         case INSTR_JMP_LT_HI:
2777         case INSTR_JMP_GT:
2778         case INSTR_JMP_GT_MH:
2779         case INSTR_JMP_GT_HM:
2780         case INSTR_JMP_GT_HH:
2781         case INSTR_JMP_GT_MI:
2782         case INSTR_JMP_GT_HI:
2783                 return 1;
2784
2785         default:
2786                 return 0;
2787         }
2788 }
2789
2790 static struct field *
2791 action_field_parse(struct action *action, const char *name);
2792
2793 static struct field *
2794 struct_field_parse(struct rte_swx_pipeline *p,
2795                    struct action *action,
2796                    const char *name,
2797                    uint32_t *struct_id)
2798 {
2799         struct field *f;
2800
2801         switch (name[0]) {
2802         case 'h':
2803         {
2804                 struct header *header;
2805
2806                 f = header_field_parse(p, name, &header);
2807                 if (!f)
2808                         return NULL;
2809
2810                 *struct_id = header->struct_id;
2811                 return f;
2812         }
2813
2814         case 'm':
2815         {
2816                 f = metadata_field_parse(p, name);
2817                 if (!f)
2818                         return NULL;
2819
2820                 *struct_id = p->metadata_struct_id;
2821                 return f;
2822         }
2823
2824         case 't':
2825         {
2826                 if (!action)
2827                         return NULL;
2828
2829                 f = action_field_parse(action, name);
2830                 if (!f)
2831                         return NULL;
2832
2833                 *struct_id = 0;
2834                 return f;
2835         }
2836
2837         case 'e':
2838         {
2839                 struct extern_obj *obj;
2840
2841                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2842                 if (!f)
2843                         return NULL;
2844
2845                 *struct_id = obj->struct_id;
2846                 return f;
2847         }
2848
2849         case 'f':
2850         {
2851                 struct extern_func *func;
2852
2853                 f = extern_func_mailbox_field_parse(p, name, &func);
2854                 if (!f)
2855                         return NULL;
2856
2857                 *struct_id = func->struct_id;
2858                 return f;
2859         }
2860
2861         default:
2862                 return NULL;
2863         }
2864 }
2865
2866 static inline void
2867 pipeline_port_inc(struct rte_swx_pipeline *p)
2868 {
2869         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2870 }
2871
2872 static inline void
2873 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2874 {
2875         t->ip = p->instructions;
2876 }
2877
2878 static inline void
2879 thread_ip_set(struct thread *t, struct instruction *ip)
2880 {
2881         t->ip = ip;
2882 }
2883
2884 static inline void
2885 thread_ip_action_call(struct rte_swx_pipeline *p,
2886                       struct thread *t,
2887                       uint32_t action_id)
2888 {
2889         t->ret = t->ip + 1;
2890         t->ip = p->action_instructions[action_id];
2891 }
2892
2893 static inline void
2894 thread_ip_inc(struct rte_swx_pipeline *p);
2895
2896 static inline void
2897 thread_ip_inc(struct rte_swx_pipeline *p)
2898 {
2899         struct thread *t = &p->threads[p->thread_id];
2900
2901         t->ip++;
2902 }
2903
2904 static inline void
2905 thread_ip_inc_cond(struct thread *t, int cond)
2906 {
2907         t->ip += cond;
2908 }
2909
2910 static inline void
2911 thread_yield(struct rte_swx_pipeline *p)
2912 {
2913         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2914 }
2915
2916 static inline void
2917 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2918 {
2919         p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2920 }
2921
2922 /*
2923  * rx.
2924  */
2925 static int
2926 instr_rx_translate(struct rte_swx_pipeline *p,
2927                    struct action *action,
2928                    char **tokens,
2929                    int n_tokens,
2930                    struct instruction *instr,
2931                    struct instruction_data *data __rte_unused)
2932 {
2933         struct field *f;
2934
2935         CHECK(!action, EINVAL);
2936         CHECK(n_tokens == 2, EINVAL);
2937
2938         f = metadata_field_parse(p, tokens[1]);
2939         CHECK(f, EINVAL);
2940
2941         instr->type = INSTR_RX;
2942         instr->io.io.offset = f->offset / 8;
2943         instr->io.io.n_bits = f->n_bits;
2944         return 0;
2945 }
2946
2947 static inline void
2948 instr_rx_exec(struct rte_swx_pipeline *p);
2949
2950 static inline void
2951 instr_rx_exec(struct rte_swx_pipeline *p)
2952 {
2953         struct thread *t = &p->threads[p->thread_id];
2954         struct instruction *ip = t->ip;
2955         struct port_in_runtime *port = &p->in[p->port_id];
2956         struct rte_swx_pkt *pkt = &t->pkt;
2957         int pkt_received;
2958
2959         /* Packet. */
2960         pkt_received = port->pkt_rx(port->obj, pkt);
2961         t->ptr = &pkt->pkt[pkt->offset];
2962         rte_prefetch0(t->ptr);
2963
2964         TRACE("[Thread %2u] rx %s from port %u\n",
2965               p->thread_id,
2966               pkt_received ? "1 pkt" : "0 pkts",
2967               p->port_id);
2968
2969         /* Headers. */
2970         t->valid_headers = 0;
2971         t->n_headers_out = 0;
2972
2973         /* Meta-data. */
2974         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2975
2976         /* Tables. */
2977         t->table_state = p->table_state;
2978
2979         /* Thread. */
2980         pipeline_port_inc(p);
2981         thread_ip_inc_cond(t, pkt_received);
2982         thread_yield(p);
2983 }
2984
2985 /*
2986  * tx.
2987  */
2988 static int
2989 instr_tx_translate(struct rte_swx_pipeline *p,
2990                    struct action *action __rte_unused,
2991                    char **tokens,
2992                    int n_tokens,
2993                    struct instruction *instr,
2994                    struct instruction_data *data __rte_unused)
2995 {
2996         char *port = tokens[1];
2997         struct field *f;
2998         uint32_t port_val;
2999
3000         CHECK(n_tokens == 2, EINVAL);
3001
3002         f = metadata_field_parse(p, port);
3003         if (f) {
3004                 instr->type = INSTR_TX;
3005                 instr->io.io.offset = f->offset / 8;
3006                 instr->io.io.n_bits = f->n_bits;
3007                 return 0;
3008         }
3009
3010         /* TX_I. */
3011         port_val = strtoul(port, &port, 0);
3012         CHECK(!port[0], EINVAL);
3013
3014         instr->type = INSTR_TX_I;
3015         instr->io.io.val = port_val;
3016         return 0;
3017 }
3018
3019 static int
3020 instr_drop_translate(struct rte_swx_pipeline *p,
3021                      struct action *action __rte_unused,
3022                      char **tokens __rte_unused,
3023                      int n_tokens,
3024                      struct instruction *instr,
3025                      struct instruction_data *data __rte_unused)
3026 {
3027         CHECK(n_tokens == 1, EINVAL);
3028
3029         /* TX_I. */
3030         instr->type = INSTR_TX_I;
3031         instr->io.io.val = p->n_ports_out - 1;
3032         return 0;
3033 }
3034
3035 static inline void
3036 emit_handler(struct thread *t)
3037 {
3038         struct header_out_runtime *h0 = &t->headers_out[0];
3039         struct header_out_runtime *h1 = &t->headers_out[1];
3040         uint32_t offset = 0, i;
3041
3042         /* No header change or header decapsulation. */
3043         if ((t->n_headers_out == 1) &&
3044             (h0->ptr + h0->n_bytes == t->ptr)) {
3045                 TRACE("Emit handler: no header change or header decap.\n");
3046
3047                 t->pkt.offset -= h0->n_bytes;
3048                 t->pkt.length += h0->n_bytes;
3049
3050                 return;
3051         }
3052
3053         /* Header encapsulation (optionally, with prior header decasulation). */
3054         if ((t->n_headers_out == 2) &&
3055             (h1->ptr + h1->n_bytes == t->ptr) &&
3056             (h0->ptr == h0->ptr0)) {
3057                 uint32_t offset;
3058
3059                 TRACE("Emit handler: header encapsulation.\n");
3060
3061                 offset = h0->n_bytes + h1->n_bytes;
3062                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
3063                 t->pkt.offset -= offset;
3064                 t->pkt.length += offset;
3065
3066                 return;
3067         }
3068
3069         /* Header insertion. */
3070         /* TBD */
3071
3072         /* Header extraction. */
3073         /* TBD */
3074
3075         /* For any other case. */
3076         TRACE("Emit handler: complex case.\n");
3077
3078         for (i = 0; i < t->n_headers_out; i++) {
3079                 struct header_out_runtime *h = &t->headers_out[i];
3080
3081                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
3082                 offset += h->n_bytes;
3083         }
3084
3085         if (offset) {
3086                 memcpy(t->ptr - offset, t->header_out_storage, offset);
3087                 t->pkt.offset -= offset;
3088                 t->pkt.length += offset;
3089         }
3090 }
3091
3092 static inline void
3093 instr_tx_exec(struct rte_swx_pipeline *p);
3094
3095 static inline void
3096 instr_tx_exec(struct rte_swx_pipeline *p)
3097 {
3098         struct thread *t = &p->threads[p->thread_id];
3099         struct instruction *ip = t->ip;
3100         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
3101         struct port_out_runtime *port = &p->out[port_id];
3102         struct rte_swx_pkt *pkt = &t->pkt;
3103
3104         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
3105               p->thread_id,
3106               (uint32_t)port_id);
3107
3108         /* Headers. */
3109         emit_handler(t);
3110
3111         /* Packet. */
3112         port->pkt_tx(port->obj, pkt);
3113
3114         /* Thread. */
3115         thread_ip_reset(p, t);
3116         instr_rx_exec(p);
3117 }
3118
3119 static inline void
3120 instr_tx_i_exec(struct rte_swx_pipeline *p)
3121 {
3122         struct thread *t = &p->threads[p->thread_id];
3123         struct instruction *ip = t->ip;
3124         uint64_t port_id = ip->io.io.val;
3125         struct port_out_runtime *port = &p->out[port_id];
3126         struct rte_swx_pkt *pkt = &t->pkt;
3127
3128         TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
3129               p->thread_id,
3130               (uint32_t)port_id);
3131
3132         /* Headers. */
3133         emit_handler(t);
3134
3135         /* Packet. */
3136         port->pkt_tx(port->obj, pkt);
3137
3138         /* Thread. */
3139         thread_ip_reset(p, t);
3140         instr_rx_exec(p);
3141 }
3142
3143 /*
3144  * extract.
3145  */
3146 static int
3147 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
3148                             struct action *action,
3149                             char **tokens,
3150                             int n_tokens,
3151                             struct instruction *instr,
3152                             struct instruction_data *data __rte_unused)
3153 {
3154         struct header *h;
3155
3156         CHECK(!action, EINVAL);
3157         CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
3158
3159         h = header_parse(p, tokens[1]);
3160         CHECK(h, EINVAL);
3161
3162         if (n_tokens == 2) {
3163                 CHECK(!h->st->var_size, EINVAL);
3164
3165                 instr->type = INSTR_HDR_EXTRACT;
3166                 instr->io.hdr.header_id[0] = h->id;
3167                 instr->io.hdr.struct_id[0] = h->struct_id;
3168                 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3169         } else {
3170                 struct field *mf;
3171
3172                 CHECK(h->st->var_size, EINVAL);
3173
3174                 mf = metadata_field_parse(p, tokens[2]);
3175                 CHECK(mf, EINVAL);
3176                 CHECK(!mf->var_size, EINVAL);
3177
3178                 instr->type = INSTR_HDR_EXTRACT_M;
3179                 instr->io.io.offset = mf->offset / 8;
3180                 instr->io.io.n_bits = mf->n_bits;
3181                 instr->io.hdr.header_id[0] = h->id;
3182                 instr->io.hdr.struct_id[0] = h->struct_id;
3183                 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
3184         }
3185
3186         return 0;
3187 }
3188
3189 static int
3190 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
3191                               struct action *action,
3192                               char **tokens,
3193                               int n_tokens,
3194                               struct instruction *instr,
3195                               struct instruction_data *data __rte_unused)
3196 {
3197         struct header *h;
3198
3199         CHECK(!action, EINVAL);
3200         CHECK(n_tokens == 2, EINVAL);
3201
3202         h = header_parse(p, tokens[1]);
3203         CHECK(h, EINVAL);
3204         CHECK(!h->st->var_size, EINVAL);
3205
3206         instr->type = INSTR_HDR_LOOKAHEAD;
3207         instr->io.hdr.header_id[0] = h->id;
3208         instr->io.hdr.struct_id[0] = h->struct_id;
3209         instr->io.hdr.n_bytes[0] = 0; /* Unused. */
3210
3211         return 0;
3212 }
3213
3214 static inline void
3215 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
3216
3217 static inline void
3218 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
3219 {
3220         struct thread *t = &p->threads[p->thread_id];
3221         struct instruction *ip = t->ip;
3222         uint64_t valid_headers = t->valid_headers;
3223         uint8_t *ptr = t->ptr;
3224         uint32_t offset = t->pkt.offset;
3225         uint32_t length = t->pkt.length;
3226         uint32_t i;
3227
3228         for (i = 0; i < n_extract; i++) {
3229                 uint32_t header_id = ip->io.hdr.header_id[i];
3230                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3231                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3232
3233                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
3234                       p->thread_id,
3235                       header_id,
3236                       n_bytes);
3237
3238                 /* Headers. */
3239                 t->structs[struct_id] = ptr;
3240                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3241
3242                 /* Packet. */
3243                 offset += n_bytes;
3244                 length -= n_bytes;
3245                 ptr += n_bytes;
3246         }
3247
3248         /* Headers. */
3249         t->valid_headers = valid_headers;
3250
3251         /* Packet. */
3252         t->pkt.offset = offset;
3253         t->pkt.length = length;
3254         t->ptr = ptr;
3255 }
3256
3257 static inline void
3258 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
3259 {
3260         __instr_hdr_extract_exec(p, 1);
3261
3262         /* Thread. */
3263         thread_ip_inc(p);
3264 }
3265
3266 static inline void
3267 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
3268 {
3269         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3270               p->thread_id);
3271
3272         __instr_hdr_extract_exec(p, 2);
3273
3274         /* Thread. */
3275         thread_ip_inc(p);
3276 }
3277
3278 static inline void
3279 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
3280 {
3281         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3282               p->thread_id);
3283
3284         __instr_hdr_extract_exec(p, 3);
3285
3286         /* Thread. */
3287         thread_ip_inc(p);
3288 }
3289
3290 static inline void
3291 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
3292 {
3293         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3294               p->thread_id);
3295
3296         __instr_hdr_extract_exec(p, 4);
3297
3298         /* Thread. */
3299         thread_ip_inc(p);
3300 }
3301
3302 static inline void
3303 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
3304 {
3305         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3306               p->thread_id);
3307
3308         __instr_hdr_extract_exec(p, 5);
3309
3310         /* Thread. */
3311         thread_ip_inc(p);
3312 }
3313
3314 static inline void
3315 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
3316 {
3317         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3318               p->thread_id);
3319
3320         __instr_hdr_extract_exec(p, 6);
3321
3322         /* Thread. */
3323         thread_ip_inc(p);
3324 }
3325
3326 static inline void
3327 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
3328 {
3329         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3330               p->thread_id);
3331
3332         __instr_hdr_extract_exec(p, 7);
3333
3334         /* Thread. */
3335         thread_ip_inc(p);
3336 }
3337
3338 static inline void
3339 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
3340 {
3341         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3342               p->thread_id);
3343
3344         __instr_hdr_extract_exec(p, 8);
3345
3346         /* Thread. */
3347         thread_ip_inc(p);
3348 }
3349
3350 static inline void
3351 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
3352 {
3353         struct thread *t = &p->threads[p->thread_id];
3354         struct instruction *ip = t->ip;
3355
3356         uint64_t valid_headers = t->valid_headers;
3357         uint8_t *ptr = t->ptr;
3358         uint32_t offset = t->pkt.offset;
3359         uint32_t length = t->pkt.length;
3360
3361         uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
3362         uint32_t header_id = ip->io.hdr.header_id[0];
3363         uint32_t struct_id = ip->io.hdr.struct_id[0];
3364         uint32_t n_bytes = ip->io.hdr.n_bytes[0];
3365
3366         struct header_runtime *h = &t->headers[header_id];
3367
3368         TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
3369               p->thread_id,
3370               header_id,
3371               n_bytes,
3372               n_bytes_last);
3373
3374         n_bytes += n_bytes_last;
3375
3376         /* Headers. */
3377         t->structs[struct_id] = ptr;
3378         t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3379         h->n_bytes = n_bytes;
3380
3381         /* Packet. */
3382         t->pkt.offset = offset + n_bytes;
3383         t->pkt.length = length - n_bytes;
3384         t->ptr = ptr + n_bytes;
3385
3386         /* Thread. */
3387         thread_ip_inc(p);
3388 }
3389
3390 static inline void
3391 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
3392 {
3393         struct thread *t = &p->threads[p->thread_id];
3394         struct instruction *ip = t->ip;
3395
3396         uint64_t valid_headers = t->valid_headers;
3397         uint8_t *ptr = t->ptr;
3398
3399         uint32_t header_id = ip->io.hdr.header_id[0];
3400         uint32_t struct_id = ip->io.hdr.struct_id[0];
3401
3402         TRACE("[Thread %2u]: lookahead header %u\n",
3403               p->thread_id,
3404               header_id);
3405
3406         /* Headers. */
3407         t->structs[struct_id] = ptr;
3408         t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3409
3410         /* Thread. */
3411         thread_ip_inc(p);
3412 }
3413
3414 /*
3415  * emit.
3416  */
3417 static int
3418 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
3419                          struct action *action __rte_unused,
3420                          char **tokens,
3421                          int n_tokens,
3422                          struct instruction *instr,
3423                          struct instruction_data *data __rte_unused)
3424 {
3425         struct header *h;
3426
3427         CHECK(n_tokens == 2, EINVAL);
3428
3429         h = header_parse(p, tokens[1]);
3430         CHECK(h, EINVAL);
3431
3432         instr->type = INSTR_HDR_EMIT;
3433         instr->io.hdr.header_id[0] = h->id;
3434         instr->io.hdr.struct_id[0] = h->struct_id;
3435         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3436         return 0;
3437 }
3438
3439 static inline void
3440 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
3441
3442 static inline void
3443 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
3444 {
3445         struct thread *t = &p->threads[p->thread_id];
3446         struct instruction *ip = t->ip;
3447         uint64_t valid_headers = t->valid_headers;
3448         uint32_t n_headers_out = t->n_headers_out;
3449         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
3450         uint8_t *ho_ptr = NULL;
3451         uint32_t ho_nbytes = 0, first = 1, i;
3452
3453         for (i = 0; i < n_emit; i++) {
3454                 uint32_t header_id = ip->io.hdr.header_id[i];
3455                 uint32_t struct_id = ip->io.hdr.struct_id[i];
3456
3457                 struct header_runtime *hi = &t->headers[header_id];
3458                 uint8_t *hi_ptr0 = hi->ptr0;
3459                 uint32_t n_bytes = hi->n_bytes;
3460
3461                 uint8_t *hi_ptr = t->structs[struct_id];
3462
3463                 if (!MASK64_BIT_GET(valid_headers, header_id))
3464                         continue;
3465
3466                 TRACE("[Thread %2u]: emit header %u\n",
3467                       p->thread_id,
3468                       header_id);
3469
3470                 /* Headers. */
3471                 if (first) {
3472                         first = 0;
3473
3474                         if (!t->n_headers_out) {
3475                                 ho = &t->headers_out[0];
3476
3477                                 ho->ptr0 = hi_ptr0;
3478                                 ho->ptr = hi_ptr;
3479
3480                                 ho_ptr = hi_ptr;
3481                                 ho_nbytes = n_bytes;
3482
3483                                 n_headers_out = 1;
3484
3485                                 continue;
3486                         } else {
3487                                 ho_ptr = ho->ptr;
3488                                 ho_nbytes = ho->n_bytes;
3489                         }
3490                 }
3491
3492                 if (ho_ptr + ho_nbytes == hi_ptr) {
3493                         ho_nbytes += n_bytes;
3494                 } else {
3495                         ho->n_bytes = ho_nbytes;
3496
3497                         ho++;
3498                         ho->ptr0 = hi_ptr0;
3499                         ho->ptr = hi_ptr;
3500
3501                         ho_ptr = hi_ptr;
3502                         ho_nbytes = n_bytes;
3503
3504                         n_headers_out++;
3505                 }
3506         }
3507
3508         ho->n_bytes = ho_nbytes;
3509         t->n_headers_out = n_headers_out;
3510 }
3511
3512 static inline void
3513 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
3514 {
3515         __instr_hdr_emit_exec(p, 1);
3516
3517         /* Thread. */
3518         thread_ip_inc(p);
3519 }
3520
3521 static inline void
3522 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
3523 {
3524         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3525               p->thread_id);
3526
3527         __instr_hdr_emit_exec(p, 1);
3528         instr_tx_exec(p);
3529 }
3530
3531 static inline void
3532 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
3533 {
3534         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3535               p->thread_id);
3536
3537         __instr_hdr_emit_exec(p, 2);
3538         instr_tx_exec(p);
3539 }
3540
3541 static inline void
3542 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
3543 {
3544         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3545               p->thread_id);
3546
3547         __instr_hdr_emit_exec(p, 3);
3548         instr_tx_exec(p);
3549 }
3550
3551 static inline void
3552 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
3553 {
3554         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3555               p->thread_id);
3556
3557         __instr_hdr_emit_exec(p, 4);
3558         instr_tx_exec(p);
3559 }
3560
3561 static inline void
3562 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
3563 {
3564         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3565               p->thread_id);
3566
3567         __instr_hdr_emit_exec(p, 5);
3568         instr_tx_exec(p);
3569 }
3570
3571 static inline void
3572 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
3573 {
3574         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3575               p->thread_id);
3576
3577         __instr_hdr_emit_exec(p, 6);
3578         instr_tx_exec(p);
3579 }
3580
3581 static inline void
3582 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
3583 {
3584         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3585               p->thread_id);
3586
3587         __instr_hdr_emit_exec(p, 7);
3588         instr_tx_exec(p);
3589 }
3590
3591 static inline void
3592 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
3593 {
3594         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
3595               p->thread_id);
3596
3597         __instr_hdr_emit_exec(p, 8);
3598         instr_tx_exec(p);
3599 }
3600
3601 /*
3602  * validate.
3603  */
3604 static int
3605 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
3606                              struct action *action __rte_unused,
3607                              char **tokens,
3608                              int n_tokens,
3609                              struct instruction *instr,
3610                              struct instruction_data *data __rte_unused)
3611 {
3612         struct header *h;
3613
3614         CHECK(n_tokens == 2, EINVAL);
3615
3616         h = header_parse(p, tokens[1]);
3617         CHECK(h, EINVAL);
3618
3619         instr->type = INSTR_HDR_VALIDATE;
3620         instr->valid.header_id = h->id;
3621         return 0;
3622 }
3623
3624 static inline void
3625 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
3626 {
3627         struct thread *t = &p->threads[p->thread_id];
3628         struct instruction *ip = t->ip;
3629         uint32_t header_id = ip->valid.header_id;
3630
3631         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
3632
3633         /* Headers. */
3634         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
3635
3636         /* Thread. */
3637         thread_ip_inc(p);
3638 }
3639
3640 /*
3641  * invalidate.
3642  */
3643 static int
3644 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
3645                                struct action *action __rte_unused,
3646                                char **tokens,
3647                                int n_tokens,
3648                                struct instruction *instr,
3649                                struct instruction_data *data __rte_unused)
3650 {
3651         struct header *h;
3652
3653         CHECK(n_tokens == 2, EINVAL);
3654
3655         h = header_parse(p, tokens[1]);
3656         CHECK(h, EINVAL);
3657
3658         instr->type = INSTR_HDR_INVALIDATE;
3659         instr->valid.header_id = h->id;
3660         return 0;
3661 }
3662
3663 static inline void
3664 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3665 {
3666         struct thread *t = &p->threads[p->thread_id];
3667         struct instruction *ip = t->ip;
3668         uint32_t header_id = ip->valid.header_id;
3669
3670         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3671
3672         /* Headers. */
3673         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3674
3675         /* Thread. */
3676         thread_ip_inc(p);
3677 }
3678
3679 /*
3680  * table.
3681  */
3682 static struct table *
3683 table_find(struct rte_swx_pipeline *p, const char *name);
3684
3685 static struct selector *
3686 selector_find(struct rte_swx_pipeline *p, const char *name);
3687
3688 static struct learner *
3689 learner_find(struct rte_swx_pipeline *p, const char *name);
3690
3691 static int
3692 instr_table_translate(struct rte_swx_pipeline *p,
3693                       struct action *action,
3694                       char **tokens,
3695                       int n_tokens,
3696                       struct instruction *instr,
3697                       struct instruction_data *data __rte_unused)
3698 {
3699         struct table *t;
3700         struct selector *s;
3701         struct learner *l;
3702
3703         CHECK(!action, EINVAL);
3704         CHECK(n_tokens == 2, EINVAL);
3705
3706         t = table_find(p, tokens[1]);
3707         if (t) {
3708                 instr->type = INSTR_TABLE;
3709                 instr->table.table_id = t->id;
3710                 return 0;
3711         }
3712
3713         s = selector_find(p, tokens[1]);
3714         if (s) {
3715                 instr->type = INSTR_SELECTOR;
3716                 instr->table.table_id = s->id;
3717                 return 0;
3718         }
3719
3720         l = learner_find(p, tokens[1]);
3721         if (l) {
3722                 instr->type = INSTR_LEARNER;
3723                 instr->table.table_id = l->id;
3724                 return 0;
3725         }
3726
3727         CHECK(0, EINVAL);
3728 }
3729
3730 static inline void
3731 instr_table_exec(struct rte_swx_pipeline *p)
3732 {
3733         struct thread *t = &p->threads[p->thread_id];
3734         struct instruction *ip = t->ip;
3735         uint32_t table_id = ip->table.table_id;
3736         struct rte_swx_table_state *ts = &t->table_state[table_id];
3737         struct table_runtime *table = &t->tables[table_id];
3738         struct table_statistics *stats = &p->table_stats[table_id];
3739         uint64_t action_id, n_pkts_hit, n_pkts_action;
3740         uint8_t *action_data;
3741         int done, hit;
3742
3743         /* Table. */
3744         done = table->func(ts->obj,
3745                            table->mailbox,
3746                            table->key,
3747                            &action_id,
3748                            &action_data,
3749                            &hit);
3750         if (!done) {
3751                 /* Thread. */
3752                 TRACE("[Thread %2u] table %u (not finalized)\n",
3753                       p->thread_id,
3754                       table_id);
3755
3756                 thread_yield(p);
3757                 return;
3758         }
3759
3760         action_id = hit ? action_id : ts->default_action_id;
3761         action_data = hit ? action_data : ts->default_action_data;
3762         n_pkts_hit = stats->n_pkts_hit[hit];
3763         n_pkts_action = stats->n_pkts_action[action_id];
3764
3765         TRACE("[Thread %2u] table %u (%s, action %u)\n",
3766               p->thread_id,
3767               table_id,
3768               hit ? "hit" : "miss",
3769               (uint32_t)action_id);
3770
3771         t->action_id = action_id;
3772         t->structs[0] = action_data;
3773         t->hit = hit;
3774         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
3775         stats->n_pkts_action[action_id] = n_pkts_action + 1;
3776
3777         /* Thread. */
3778         thread_ip_action_call(p, t, action_id);
3779 }
3780
3781 static inline void
3782 instr_selector_exec(struct rte_swx_pipeline *p)
3783 {
3784         struct thread *t = &p->threads[p->thread_id];
3785         struct instruction *ip = t->ip;
3786         uint32_t selector_id = ip->table.table_id;
3787         struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
3788         struct selector_runtime *selector = &t->selectors[selector_id];
3789         struct selector_statistics *stats = &p->selector_stats[selector_id];
3790         uint64_t n_pkts = stats->n_pkts;
3791         int done;
3792
3793         /* Table. */
3794         done = rte_swx_table_selector_select(ts->obj,
3795                            selector->mailbox,
3796                            selector->group_id_buffer,
3797                            selector->selector_buffer,
3798                            selector->member_id_buffer);
3799         if (!done) {
3800                 /* Thread. */
3801                 TRACE("[Thread %2u] selector %u (not finalized)\n",
3802                       p->thread_id,
3803                       selector_id);
3804
3805                 thread_yield(p);
3806                 return;
3807         }
3808
3809
3810         TRACE("[Thread %2u] selector %u\n",
3811               p->thread_id,
3812               selector_id);
3813
3814         stats->n_pkts = n_pkts + 1;
3815
3816         /* Thread. */
3817         thread_ip_inc(p);
3818 }
3819
3820 static inline void
3821 instr_learner_exec(struct rte_swx_pipeline *p)
3822 {
3823         struct thread *t = &p->threads[p->thread_id];
3824         struct instruction *ip = t->ip;
3825         uint32_t learner_id = ip->table.table_id;
3826         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
3827                 p->n_selectors + learner_id];
3828         struct learner_runtime *l = &t->learners[learner_id];
3829         struct learner_statistics *stats = &p->learner_stats[learner_id];
3830         uint64_t action_id, n_pkts_hit, n_pkts_action, time;
3831         uint8_t *action_data;
3832         int done, hit;
3833
3834         /* Table. */
3835         time = rte_get_tsc_cycles();
3836
3837         done = rte_swx_table_learner_lookup(ts->obj,
3838                                             l->mailbox,
3839                                             time,
3840                                             l->key,
3841                                             &action_id,
3842                                             &action_data,
3843                                             &hit);
3844         if (!done) {
3845                 /* Thread. */
3846                 TRACE("[Thread %2u] learner %u (not finalized)\n",
3847                       p->thread_id,
3848                       learner_id);
3849
3850                 thread_yield(p);
3851                 return;
3852         }
3853
3854         action_id = hit ? action_id : ts->default_action_id;
3855         action_data = hit ? action_data : ts->default_action_data;
3856         n_pkts_hit = stats->n_pkts_hit[hit];
3857         n_pkts_action = stats->n_pkts_action[action_id];
3858
3859         TRACE("[Thread %2u] learner %u (%s, action %u)\n",
3860               p->thread_id,
3861               learner_id,
3862               hit ? "hit" : "miss",
3863               (uint32_t)action_id);
3864
3865         t->action_id = action_id;
3866         t->structs[0] = action_data;
3867         t->hit = hit;
3868         t->learner_id = learner_id;
3869         t->time = time;
3870         stats->n_pkts_hit[hit] = n_pkts_hit + 1;
3871         stats->n_pkts_action[action_id] = n_pkts_action + 1;
3872
3873         /* Thread. */
3874         thread_ip_action_call(p, t, action_id);
3875 }
3876
3877 /*
3878  * learn.
3879  */
3880 static struct action *
3881 action_find(struct rte_swx_pipeline *p, const char *name);
3882
3883 static int
3884 action_has_nbo_args(struct action *a);
3885
3886 static int
3887 instr_learn_translate(struct rte_swx_pipeline *p,
3888                       struct action *action,
3889                       char **tokens,
3890                       int n_tokens,
3891                       struct instruction *instr,
3892                       struct instruction_data *data __rte_unused)
3893 {
3894         struct action *a;
3895
3896         CHECK(action, EINVAL);
3897         CHECK(n_tokens == 2, EINVAL);
3898
3899         a = action_find(p, tokens[1]);
3900         CHECK(a, EINVAL);
3901         CHECK(!action_has_nbo_args(a), EINVAL);
3902
3903         instr->type = INSTR_LEARNER_LEARN;
3904         instr->learn.action_id = a->id;
3905
3906         return 0;
3907 }
3908
3909 static inline void
3910 instr_learn_exec(struct rte_swx_pipeline *p)
3911 {
3912         struct thread *t = &p->threads[p->thread_id];
3913         struct instruction *ip = t->ip;
3914         uint64_t action_id = ip->learn.action_id;
3915         uint32_t learner_id = t->learner_id;
3916         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
3917                 p->n_selectors + learner_id];
3918         struct learner_runtime *l = &t->learners[learner_id];
3919         struct learner_statistics *stats = &p->learner_stats[learner_id];
3920         uint32_t status;
3921
3922         /* Table. */
3923         status = rte_swx_table_learner_add(ts->obj,
3924                                            l->mailbox,
3925                                            t->time,
3926                                            action_id,
3927                                            l->action_data[action_id]);
3928
3929         TRACE("[Thread %2u] learner %u learn %s\n",
3930               p->thread_id,
3931               learner_id,
3932               status ? "ok" : "error");
3933
3934         stats->n_pkts_learn[status] += 1;
3935
3936         /* Thread. */
3937         thread_ip_inc(p);
3938 }
3939
3940 /*
3941  * forget.
3942  */
3943 static int
3944 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
3945                        struct action *action,
3946                        char **tokens __rte_unused,
3947                        int n_tokens,
3948                        struct instruction *instr,
3949                        struct instruction_data *data __rte_unused)
3950 {
3951         CHECK(action, EINVAL);
3952         CHECK(n_tokens == 1, EINVAL);
3953
3954         instr->type = INSTR_LEARNER_FORGET;
3955
3956         return 0;
3957 }
3958
3959 static inline void
3960 instr_forget_exec(struct rte_swx_pipeline *p)
3961 {
3962         struct thread *t = &p->threads[p->thread_id];
3963         uint32_t learner_id = t->learner_id;
3964         struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
3965                 p->n_selectors + learner_id];
3966         struct learner_runtime *l = &t->learners[learner_id];
3967         struct learner_statistics *stats = &p->learner_stats[learner_id];
3968
3969         /* Table. */
3970         rte_swx_table_learner_delete(ts->obj, l->mailbox);
3971
3972         TRACE("[Thread %2u] learner %u forget\n",
3973               p->thread_id,
3974               learner_id);
3975
3976         stats->n_pkts_forget += 1;
3977
3978         /* Thread. */
3979         thread_ip_inc(p);
3980 }
3981
3982 /*
3983  * extern.
3984  */
3985 static int
3986 instr_extern_translate(struct rte_swx_pipeline *p,
3987                        struct action *action __rte_unused,
3988                        char **tokens,
3989                        int n_tokens,
3990                        struct instruction *instr,
3991                        struct instruction_data *data __rte_unused)
3992 {
3993         char *token = tokens[1];
3994
3995         CHECK(n_tokens == 2, EINVAL);
3996
3997         if (token[0] == 'e') {
3998                 struct extern_obj *obj;
3999                 struct extern_type_member_func *func;
4000
4001                 func = extern_obj_member_func_parse(p, token, &obj);
4002                 CHECK(func, EINVAL);
4003
4004                 instr->type = INSTR_EXTERN_OBJ;
4005                 instr->ext_obj.ext_obj_id = obj->id;
4006                 instr->ext_obj.func_id = func->id;
4007
4008                 return 0;
4009         }
4010
4011         if (token[0] == 'f') {
4012                 struct extern_func *func;
4013
4014                 func = extern_func_parse(p, token);
4015                 CHECK(func, EINVAL);
4016
4017                 instr->type = INSTR_EXTERN_FUNC;
4018                 instr->ext_func.ext_func_id = func->id;
4019
4020                 return 0;
4021         }
4022
4023         CHECK(0, EINVAL);
4024 }
4025
4026 static inline void
4027 instr_extern_obj_exec(struct rte_swx_pipeline *p)
4028 {
4029         struct thread *t = &p->threads[p->thread_id];
4030         struct instruction *ip = t->ip;
4031         uint32_t obj_id = ip->ext_obj.ext_obj_id;
4032         uint32_t func_id = ip->ext_obj.func_id;
4033         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
4034         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
4035
4036         TRACE("[Thread %2u] extern obj %u member func %u\n",
4037               p->thread_id,
4038               obj_id,
4039               func_id);
4040
4041         /* Extern object member function execute. */
4042         uint32_t done = func(obj->obj, obj->mailbox);
4043
4044         /* Thread. */
4045         thread_ip_inc_cond(t, done);
4046         thread_yield_cond(p, done ^ 1);
4047 }
4048
4049 static inline void
4050 instr_extern_func_exec(struct rte_swx_pipeline *p)
4051 {
4052         struct thread *t = &p->threads[p->thread_id];
4053         struct instruction *ip = t->ip;
4054         uint32_t ext_func_id = ip->ext_func.ext_func_id;
4055         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
4056         rte_swx_extern_func_t func = ext_func->func;
4057
4058         TRACE("[Thread %2u] extern func %u\n",
4059               p->thread_id,
4060               ext_func_id);
4061
4062         /* Extern function execute. */
4063         uint32_t done = func(ext_func->mailbox);
4064
4065         /* Thread. */
4066         thread_ip_inc_cond(t, done);
4067         thread_yield_cond(p, done ^ 1);
4068 }
4069
4070 /*
4071  * mov.
4072  */
4073 static int
4074 instr_mov_translate(struct rte_swx_pipeline *p,
4075                     struct action *action,
4076                     char **tokens,
4077                     int n_tokens,
4078                     struct instruction *instr,
4079                     struct instruction_data *data __rte_unused)
4080 {
4081         char *dst = tokens[1], *src = tokens[2];
4082         struct field *fdst, *fsrc;
4083         uint64_t src_val;
4084         uint32_t dst_struct_id = 0, src_struct_id = 0;
4085
4086         CHECK(n_tokens == 3, EINVAL);
4087
4088         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4089         CHECK(fdst, EINVAL);
4090         CHECK(!fdst->var_size, EINVAL);
4091
4092         /* MOV, MOV_MH, MOV_HM or MOV_HH. */
4093         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4094         if (fsrc) {
4095                 CHECK(!fsrc->var_size, EINVAL);
4096
4097                 instr->type = INSTR_MOV;
4098                 if (dst[0] != 'h' && src[0] == 'h')
4099                         instr->type = INSTR_MOV_MH;
4100                 if (dst[0] == 'h' && src[0] != 'h')
4101                         instr->type = INSTR_MOV_HM;
4102                 if (dst[0] == 'h' && src[0] == 'h')
4103                         instr->type = INSTR_MOV_HH;
4104
4105                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
4106                 instr->mov.dst.n_bits = fdst->n_bits;
4107                 instr->mov.dst.offset = fdst->offset / 8;
4108                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
4109                 instr->mov.src.n_bits = fsrc->n_bits;
4110                 instr->mov.src.offset = fsrc->offset / 8;
4111                 return 0;
4112         }
4113
4114         /* MOV_I. */
4115         src_val = strtoull(src, &src, 0);
4116         CHECK(!src[0], EINVAL);
4117
4118         if (dst[0] == 'h')
4119                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4120
4121         instr->type = INSTR_MOV_I;
4122         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
4123         instr->mov.dst.n_bits = fdst->n_bits;
4124         instr->mov.dst.offset = fdst->offset / 8;
4125         instr->mov.src_val = src_val;
4126         return 0;
4127 }
4128
4129 static inline void
4130 instr_mov_exec(struct rte_swx_pipeline *p)
4131 {
4132         struct thread *t = &p->threads[p->thread_id];
4133         struct instruction *ip = t->ip;
4134
4135         TRACE("[Thread %2u] mov\n",
4136               p->thread_id);
4137
4138         MOV(t, ip);
4139
4140         /* Thread. */
4141         thread_ip_inc(p);
4142 }
4143
4144 static inline void
4145 instr_mov_mh_exec(struct rte_swx_pipeline *p)
4146 {
4147         struct thread *t = &p->threads[p->thread_id];
4148         struct instruction *ip = t->ip;
4149
4150         TRACE("[Thread %2u] mov (mh)\n",
4151               p->thread_id);
4152
4153         MOV_MH(t, ip);
4154
4155         /* Thread. */
4156         thread_ip_inc(p);
4157 }
4158
4159 static inline void
4160 instr_mov_hm_exec(struct rte_swx_pipeline *p)
4161 {
4162         struct thread *t = &p->threads[p->thread_id];
4163         struct instruction *ip = t->ip;
4164
4165         TRACE("[Thread %2u] mov (hm)\n",
4166               p->thread_id);
4167
4168         MOV_HM(t, ip);
4169
4170         /* Thread. */
4171         thread_ip_inc(p);
4172 }
4173
4174 static inline void
4175 instr_mov_hh_exec(struct rte_swx_pipeline *p)
4176 {
4177         struct thread *t = &p->threads[p->thread_id];
4178         struct instruction *ip = t->ip;
4179
4180         TRACE("[Thread %2u] mov (hh)\n",
4181               p->thread_id);
4182
4183         MOV_HH(t, ip);
4184
4185         /* Thread. */
4186         thread_ip_inc(p);
4187 }
4188
4189 static inline void
4190 instr_mov_i_exec(struct rte_swx_pipeline *p)
4191 {
4192         struct thread *t = &p->threads[p->thread_id];
4193         struct instruction *ip = t->ip;
4194
4195         TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
4196               p->thread_id,
4197               ip->mov.src_val);
4198
4199         MOV_I(t, ip);
4200
4201         /* Thread. */
4202         thread_ip_inc(p);
4203 }
4204
4205 /*
4206  * dma.
4207  */
4208 static inline void
4209 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
4210
4211 static inline void
4212 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
4213 {
4214         struct thread *t = &p->threads[p->thread_id];
4215         struct instruction *ip = t->ip;
4216         uint8_t *action_data = t->structs[0];
4217         uint64_t valid_headers = t->valid_headers;
4218         uint32_t i;
4219
4220         for (i = 0; i < n_dma; i++) {
4221                 uint32_t header_id = ip->dma.dst.header_id[i];
4222                 uint32_t struct_id = ip->dma.dst.struct_id[i];
4223                 uint32_t offset = ip->dma.src.offset[i];
4224                 uint32_t n_bytes = ip->dma.n_bytes[i];
4225
4226                 struct header_runtime *h = &t->headers[header_id];
4227                 uint8_t *h_ptr0 = h->ptr0;
4228                 uint8_t *h_ptr = t->structs[struct_id];
4229
4230                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
4231                         h_ptr : h_ptr0;
4232                 void *src = &action_data[offset];
4233
4234                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
4235
4236                 /* Headers. */
4237                 memcpy(dst, src, n_bytes);
4238                 t->structs[struct_id] = dst;
4239                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
4240         }
4241
4242         t->valid_headers = valid_headers;
4243 }
4244
4245 static inline void
4246 instr_dma_ht_exec(struct rte_swx_pipeline *p)
4247 {
4248         __instr_dma_ht_exec(p, 1);
4249
4250         /* Thread. */
4251         thread_ip_inc(p);
4252 }
4253
4254 static inline void
4255 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
4256 {
4257         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
4258               p->thread_id);
4259
4260         __instr_dma_ht_exec(p, 2);
4261
4262         /* Thread. */
4263         thread_ip_inc(p);
4264 }
4265
4266 static inline void
4267 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
4268 {
4269         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
4270               p->thread_id);
4271
4272         __instr_dma_ht_exec(p, 3);
4273
4274         /* Thread. */
4275         thread_ip_inc(p);
4276 }
4277
4278 static inline void
4279 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
4280 {
4281         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
4282               p->thread_id);
4283
4284         __instr_dma_ht_exec(p, 4);
4285
4286         /* Thread. */
4287         thread_ip_inc(p);
4288 }
4289
4290 static inline void
4291 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
4292 {
4293         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
4294               p->thread_id);
4295
4296         __instr_dma_ht_exec(p, 5);
4297
4298         /* Thread. */
4299         thread_ip_inc(p);
4300 }
4301
4302 static inline void
4303 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
4304 {
4305         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
4306               p->thread_id);
4307
4308         __instr_dma_ht_exec(p, 6);
4309
4310         /* Thread. */
4311         thread_ip_inc(p);
4312 }
4313
4314 static inline void
4315 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
4316 {
4317         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
4318               p->thread_id);
4319
4320         __instr_dma_ht_exec(p, 7);
4321
4322         /* Thread. */
4323         thread_ip_inc(p);
4324 }
4325
4326 static inline void
4327 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
4328 {
4329         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
4330               p->thread_id);
4331
4332         __instr_dma_ht_exec(p, 8);
4333
4334         /* Thread. */
4335         thread_ip_inc(p);
4336 }
4337
4338 /*
4339  * alu.
4340  */
4341 static int
4342 instr_alu_add_translate(struct rte_swx_pipeline *p,
4343                         struct action *action,
4344                         char **tokens,
4345                         int n_tokens,
4346                         struct instruction *instr,
4347                         struct instruction_data *data __rte_unused)
4348 {
4349         char *dst = tokens[1], *src = tokens[2];
4350         struct field *fdst, *fsrc;
4351         uint64_t src_val;
4352         uint32_t dst_struct_id = 0, src_struct_id = 0;
4353
4354         CHECK(n_tokens == 3, EINVAL);
4355
4356         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4357         CHECK(fdst, EINVAL);
4358         CHECK(!fdst->var_size, EINVAL);
4359
4360         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
4361         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4362         if (fsrc) {
4363                 CHECK(!fsrc->var_size, EINVAL);
4364
4365                 instr->type = INSTR_ALU_ADD;
4366                 if (dst[0] == 'h' && src[0] != 'h')
4367                         instr->type = INSTR_ALU_ADD_HM;
4368                 if (dst[0] != 'h' && src[0] == 'h')
4369                         instr->type = INSTR_ALU_ADD_MH;
4370                 if (dst[0] == 'h' && src[0] == 'h')
4371                         instr->type = INSTR_ALU_ADD_HH;
4372
4373                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4374                 instr->alu.dst.n_bits = fdst->n_bits;
4375                 instr->alu.dst.offset = fdst->offset / 8;
4376                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4377                 instr->alu.src.n_bits = fsrc->n_bits;
4378                 instr->alu.src.offset = fsrc->offset / 8;
4379                 return 0;
4380         }
4381
4382         /* ADD_MI, ADD_HI. */
4383         src_val = strtoull(src, &src, 0);
4384         CHECK(!src[0], EINVAL);
4385
4386         instr->type = INSTR_ALU_ADD_MI;
4387         if (dst[0] == 'h')
4388                 instr->type = INSTR_ALU_ADD_HI;
4389
4390         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4391         instr->alu.dst.n_bits = fdst->n_bits;
4392         instr->alu.dst.offset = fdst->offset / 8;
4393         instr->alu.src_val = src_val;
4394         return 0;
4395 }
4396
4397 static int
4398 instr_alu_sub_translate(struct rte_swx_pipeline *p,
4399                         struct action *action,
4400                         char **tokens,
4401                         int n_tokens,
4402                         struct instruction *instr,
4403                         struct instruction_data *data __rte_unused)
4404 {
4405         char *dst = tokens[1], *src = tokens[2];
4406         struct field *fdst, *fsrc;
4407         uint64_t src_val;
4408         uint32_t dst_struct_id = 0, src_struct_id = 0;
4409
4410         CHECK(n_tokens == 3, EINVAL);
4411
4412         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4413         CHECK(fdst, EINVAL);
4414         CHECK(!fdst->var_size, EINVAL);
4415
4416         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
4417         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4418         if (fsrc) {
4419                 CHECK(!fsrc->var_size, EINVAL);
4420
4421                 instr->type = INSTR_ALU_SUB;
4422                 if (dst[0] == 'h' && src[0] != 'h')
4423                         instr->type = INSTR_ALU_SUB_HM;
4424                 if (dst[0] != 'h' && src[0] == 'h')
4425                         instr->type = INSTR_ALU_SUB_MH;
4426                 if (dst[0] == 'h' && src[0] == 'h')
4427                         instr->type = INSTR_ALU_SUB_HH;
4428
4429                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4430                 instr->alu.dst.n_bits = fdst->n_bits;
4431                 instr->alu.dst.offset = fdst->offset / 8;
4432                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4433                 instr->alu.src.n_bits = fsrc->n_bits;
4434                 instr->alu.src.offset = fsrc->offset / 8;
4435                 return 0;
4436         }
4437
4438         /* SUB_MI, SUB_HI. */
4439         src_val = strtoull(src, &src, 0);
4440         CHECK(!src[0], EINVAL);
4441
4442         instr->type = INSTR_ALU_SUB_MI;
4443         if (dst[0] == 'h')
4444                 instr->type = INSTR_ALU_SUB_HI;
4445
4446         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4447         instr->alu.dst.n_bits = fdst->n_bits;
4448         instr->alu.dst.offset = fdst->offset / 8;
4449         instr->alu.src_val = src_val;
4450         return 0;
4451 }
4452
4453 static int
4454 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
4455                           struct action *action __rte_unused,
4456                           char **tokens,
4457                           int n_tokens,
4458                           struct instruction *instr,
4459                           struct instruction_data *data __rte_unused)
4460 {
4461         char *dst = tokens[1], *src = tokens[2];
4462         struct header *hdst, *hsrc;
4463         struct field *fdst, *fsrc;
4464
4465         CHECK(n_tokens == 3, EINVAL);
4466
4467         fdst = header_field_parse(p, dst, &hdst);
4468         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4469         CHECK(!fdst->var_size, EINVAL);
4470
4471         /* CKADD_FIELD. */
4472         fsrc = header_field_parse(p, src, &hsrc);
4473         if (fsrc) {
4474                 CHECK(!fsrc->var_size, EINVAL);
4475
4476                 instr->type = INSTR_ALU_CKADD_FIELD;
4477                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4478                 instr->alu.dst.n_bits = fdst->n_bits;
4479                 instr->alu.dst.offset = fdst->offset / 8;
4480                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4481                 instr->alu.src.n_bits = fsrc->n_bits;
4482                 instr->alu.src.offset = fsrc->offset / 8;
4483                 return 0;
4484         }
4485
4486         /* CKADD_STRUCT, CKADD_STRUCT20. */
4487         hsrc = header_parse(p, src);
4488         CHECK(hsrc, EINVAL);
4489         CHECK(!hsrc->st->var_size, EINVAL);
4490
4491         instr->type = INSTR_ALU_CKADD_STRUCT;
4492         if ((hsrc->st->n_bits / 8) == 20)
4493                 instr->type = INSTR_ALU_CKADD_STRUCT20;
4494
4495         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4496         instr->alu.dst.n_bits = fdst->n_bits;
4497         instr->alu.dst.offset = fdst->offset / 8;
4498         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4499         instr->alu.src.n_bits = hsrc->st->n_bits;
4500         instr->alu.src.offset = 0; /* Unused. */
4501         return 0;
4502 }
4503
4504 static int
4505 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
4506                           struct action *action __rte_unused,
4507                           char **tokens,
4508                           int n_tokens,
4509                           struct instruction *instr,
4510                           struct instruction_data *data __rte_unused)
4511 {
4512         char *dst = tokens[1], *src = tokens[2];
4513         struct header *hdst, *hsrc;
4514         struct field *fdst, *fsrc;
4515
4516         CHECK(n_tokens == 3, EINVAL);
4517
4518         fdst = header_field_parse(p, dst, &hdst);
4519         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4520         CHECK(!fdst->var_size, EINVAL);
4521
4522         fsrc = header_field_parse(p, src, &hsrc);
4523         CHECK(fsrc, EINVAL);
4524         CHECK(!fsrc->var_size, EINVAL);
4525
4526         instr->type = INSTR_ALU_CKSUB_FIELD;
4527         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4528         instr->alu.dst.n_bits = fdst->n_bits;
4529         instr->alu.dst.offset = fdst->offset / 8;
4530         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4531         instr->alu.src.n_bits = fsrc->n_bits;
4532         instr->alu.src.offset = fsrc->offset / 8;
4533         return 0;
4534 }
4535
4536 static int
4537 instr_alu_shl_translate(struct rte_swx_pipeline *p,
4538                         struct action *action,
4539                         char **tokens,
4540                         int n_tokens,
4541                         struct instruction *instr,
4542                         struct instruction_data *data __rte_unused)
4543 {
4544         char *dst = tokens[1], *src = tokens[2];
4545         struct field *fdst, *fsrc;
4546         uint64_t src_val;
4547         uint32_t dst_struct_id = 0, src_struct_id = 0;
4548
4549         CHECK(n_tokens == 3, EINVAL);
4550
4551         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4552         CHECK(fdst, EINVAL);
4553         CHECK(!fdst->var_size, EINVAL);
4554
4555         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
4556         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4557         if (fsrc) {
4558                 CHECK(!fsrc->var_size, EINVAL);
4559
4560                 instr->type = INSTR_ALU_SHL;
4561                 if (dst[0] == 'h' && src[0] != 'h')
4562                         instr->type = INSTR_ALU_SHL_HM;
4563                 if (dst[0] != 'h' && src[0] == 'h')
4564                         instr->type = INSTR_ALU_SHL_MH;
4565                 if (dst[0] == 'h' && src[0] == 'h')
4566                         instr->type = INSTR_ALU_SHL_HH;
4567
4568                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4569                 instr->alu.dst.n_bits = fdst->n_bits;
4570                 instr->alu.dst.offset = fdst->offset / 8;
4571                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4572                 instr->alu.src.n_bits = fsrc->n_bits;
4573                 instr->alu.src.offset = fsrc->offset / 8;
4574                 return 0;
4575         }
4576
4577         /* SHL_MI, SHL_HI. */
4578         src_val = strtoull(src, &src, 0);
4579         CHECK(!src[0], EINVAL);
4580
4581         instr->type = INSTR_ALU_SHL_MI;
4582         if (dst[0] == 'h')
4583                 instr->type = INSTR_ALU_SHL_HI;
4584
4585         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4586         instr->alu.dst.n_bits = fdst->n_bits;
4587         instr->alu.dst.offset = fdst->offset / 8;
4588         instr->alu.src_val = src_val;
4589         return 0;
4590 }
4591
4592 static int
4593 instr_alu_shr_translate(struct rte_swx_pipeline *p,
4594                         struct action *action,
4595                         char **tokens,
4596                         int n_tokens,
4597                         struct instruction *instr,
4598                         struct instruction_data *data __rte_unused)
4599 {
4600         char *dst = tokens[1], *src = tokens[2];
4601         struct field *fdst, *fsrc;
4602         uint64_t src_val;
4603         uint32_t dst_struct_id = 0, src_struct_id = 0;
4604
4605         CHECK(n_tokens == 3, EINVAL);
4606
4607         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4608         CHECK(fdst, EINVAL);
4609         CHECK(!fdst->var_size, EINVAL);
4610
4611         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
4612         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4613         if (fsrc) {
4614                 CHECK(!fsrc->var_size, EINVAL);
4615
4616                 instr->type = INSTR_ALU_SHR;
4617                 if (dst[0] == 'h' && src[0] != 'h')
4618                         instr->type = INSTR_ALU_SHR_HM;
4619                 if (dst[0] != 'h' && src[0] == 'h')
4620                         instr->type = INSTR_ALU_SHR_MH;
4621                 if (dst[0] == 'h' && src[0] == 'h')
4622                         instr->type = INSTR_ALU_SHR_HH;
4623
4624                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4625                 instr->alu.dst.n_bits = fdst->n_bits;
4626                 instr->alu.dst.offset = fdst->offset / 8;
4627                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4628                 instr->alu.src.n_bits = fsrc->n_bits;
4629                 instr->alu.src.offset = fsrc->offset / 8;
4630                 return 0;
4631         }
4632
4633         /* SHR_MI, SHR_HI. */
4634         src_val = strtoull(src, &src, 0);
4635         CHECK(!src[0], EINVAL);
4636
4637         instr->type = INSTR_ALU_SHR_MI;
4638         if (dst[0] == 'h')
4639                 instr->type = INSTR_ALU_SHR_HI;
4640
4641         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4642         instr->alu.dst.n_bits = fdst->n_bits;
4643         instr->alu.dst.offset = fdst->offset / 8;
4644         instr->alu.src_val = src_val;
4645         return 0;
4646 }
4647
4648 static int
4649 instr_alu_and_translate(struct rte_swx_pipeline *p,
4650                         struct action *action,
4651                         char **tokens,
4652                         int n_tokens,
4653                         struct instruction *instr,
4654                         struct instruction_data *data __rte_unused)
4655 {
4656         char *dst = tokens[1], *src = tokens[2];
4657         struct field *fdst, *fsrc;
4658         uint64_t src_val;
4659         uint32_t dst_struct_id = 0, src_struct_id = 0;
4660
4661         CHECK(n_tokens == 3, EINVAL);
4662
4663         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4664         CHECK(fdst, EINVAL);
4665         CHECK(!fdst->var_size, EINVAL);
4666
4667         /* AND, AND_MH, AND_HM, AND_HH. */
4668         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4669         if (fsrc) {
4670                 CHECK(!fsrc->var_size, EINVAL);
4671
4672                 instr->type = INSTR_ALU_AND;
4673                 if (dst[0] != 'h' && src[0] == 'h')
4674                         instr->type = INSTR_ALU_AND_MH;
4675                 if (dst[0] == 'h' && src[0] != 'h')
4676                         instr->type = INSTR_ALU_AND_HM;
4677                 if (dst[0] == 'h' && src[0] == 'h')
4678                         instr->type = INSTR_ALU_AND_HH;
4679
4680                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4681                 instr->alu.dst.n_bits = fdst->n_bits;
4682                 instr->alu.dst.offset = fdst->offset / 8;
4683                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4684                 instr->alu.src.n_bits = fsrc->n_bits;
4685                 instr->alu.src.offset = fsrc->offset / 8;
4686                 return 0;
4687         }
4688
4689         /* AND_I. */
4690         src_val = strtoull(src, &src, 0);
4691         CHECK(!src[0], EINVAL);
4692
4693         if (dst[0] == 'h')
4694                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4695
4696         instr->type = INSTR_ALU_AND_I;
4697         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4698         instr->alu.dst.n_bits = fdst->n_bits;
4699         instr->alu.dst.offset = fdst->offset / 8;
4700         instr->alu.src_val = src_val;
4701         return 0;
4702 }
4703
4704 static int
4705 instr_alu_or_translate(struct rte_swx_pipeline *p,
4706                        struct action *action,
4707                        char **tokens,
4708                        int n_tokens,
4709                        struct instruction *instr,
4710                        struct instruction_data *data __rte_unused)
4711 {
4712         char *dst = tokens[1], *src = tokens[2];
4713         struct field *fdst, *fsrc;
4714         uint64_t src_val;
4715         uint32_t dst_struct_id = 0, src_struct_id = 0;
4716
4717         CHECK(n_tokens == 3, EINVAL);
4718
4719         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4720         CHECK(fdst, EINVAL);
4721         CHECK(!fdst->var_size, EINVAL);
4722
4723         /* OR, OR_MH, OR_HM, OR_HH. */
4724         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4725         if (fsrc) {
4726                 CHECK(!fsrc->var_size, EINVAL);
4727
4728                 instr->type = INSTR_ALU_OR;
4729                 if (dst[0] != 'h' && src[0] == 'h')
4730                         instr->type = INSTR_ALU_OR_MH;
4731                 if (dst[0] == 'h' && src[0] != 'h')
4732                         instr->type = INSTR_ALU_OR_HM;
4733                 if (dst[0] == 'h' && src[0] == 'h')
4734                         instr->type = INSTR_ALU_OR_HH;
4735
4736                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4737                 instr->alu.dst.n_bits = fdst->n_bits;
4738                 instr->alu.dst.offset = fdst->offset / 8;
4739                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4740                 instr->alu.src.n_bits = fsrc->n_bits;
4741                 instr->alu.src.offset = fsrc->offset / 8;
4742                 return 0;
4743         }
4744
4745         /* OR_I. */
4746         src_val = strtoull(src, &src, 0);
4747         CHECK(!src[0], EINVAL);
4748
4749         if (dst[0] == 'h')
4750                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4751
4752         instr->type = INSTR_ALU_OR_I;
4753         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4754         instr->alu.dst.n_bits = fdst->n_bits;
4755         instr->alu.dst.offset = fdst->offset / 8;
4756         instr->alu.src_val = src_val;
4757         return 0;
4758 }
4759
4760 static int
4761 instr_alu_xor_translate(struct rte_swx_pipeline *p,
4762                         struct action *action,
4763                         char **tokens,
4764                         int n_tokens,
4765                         struct instruction *instr,
4766                         struct instruction_data *data __rte_unused)
4767 {
4768         char *dst = tokens[1], *src = tokens[2];
4769         struct field *fdst, *fsrc;
4770         uint64_t src_val;
4771         uint32_t dst_struct_id = 0, src_struct_id = 0;
4772
4773         CHECK(n_tokens == 3, EINVAL);
4774
4775         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4776         CHECK(fdst, EINVAL);
4777         CHECK(!fdst->var_size, EINVAL);
4778
4779         /* XOR, XOR_MH, XOR_HM, XOR_HH. */
4780         fsrc = struct_field_parse(p, action, src, &src_struct_id);
4781         if (fsrc) {
4782                 CHECK(!fsrc->var_size, EINVAL);
4783
4784                 instr->type = INSTR_ALU_XOR;
4785                 if (dst[0] != 'h' && src[0] == 'h')
4786                         instr->type = INSTR_ALU_XOR_MH;
4787                 if (dst[0] == 'h' && src[0] != 'h')
4788                         instr->type = INSTR_ALU_XOR_HM;
4789                 if (dst[0] == 'h' && src[0] == 'h')
4790                         instr->type = INSTR_ALU_XOR_HH;
4791
4792                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4793                 instr->alu.dst.n_bits = fdst->n_bits;
4794                 instr->alu.dst.offset = fdst->offset / 8;
4795                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4796                 instr->alu.src.n_bits = fsrc->n_bits;
4797                 instr->alu.src.offset = fsrc->offset / 8;
4798                 return 0;
4799         }
4800
4801         /* XOR_I. */
4802         src_val = strtoull(src, &src, 0);
4803         CHECK(!src[0], EINVAL);
4804
4805         if (dst[0] == 'h')
4806                 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4807
4808         instr->type = INSTR_ALU_XOR_I;
4809         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4810         instr->alu.dst.n_bits = fdst->n_bits;
4811         instr->alu.dst.offset = fdst->offset / 8;
4812         instr->alu.src_val = src_val;
4813         return 0;
4814 }
4815
4816 static inline void
4817 instr_alu_add_exec(struct rte_swx_pipeline *p)
4818 {
4819         struct thread *t = &p->threads[p->thread_id];
4820         struct instruction *ip = t->ip;
4821
4822         TRACE("[Thread %2u] add\n", p->thread_id);
4823
4824         /* Structs. */
4825         ALU(t, ip, +);
4826
4827         /* Thread. */
4828         thread_ip_inc(p);
4829 }
4830
4831 static inline void
4832 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
4833 {
4834         struct thread *t = &p->threads[p->thread_id];
4835         struct instruction *ip = t->ip;
4836
4837         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
4838
4839         /* Structs. */
4840         ALU_MH(t, ip, +);
4841
4842         /* Thread. */
4843         thread_ip_inc(p);
4844 }
4845
4846 static inline void
4847 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
4848 {
4849         struct thread *t = &p->threads[p->thread_id];
4850         struct instruction *ip = t->ip;
4851
4852         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
4853
4854         /* Structs. */
4855         ALU_HM(t, ip, +);
4856
4857         /* Thread. */
4858         thread_ip_inc(p);
4859 }
4860
4861 static inline void
4862 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
4863 {
4864         struct thread *t = &p->threads[p->thread_id];
4865         struct instruction *ip = t->ip;
4866
4867         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
4868
4869         /* Structs. */
4870         ALU_HH(t, ip, +);
4871
4872         /* Thread. */
4873         thread_ip_inc(p);
4874 }
4875
4876 static inline void
4877 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
4878 {
4879         struct thread *t = &p->threads[p->thread_id];
4880         struct instruction *ip = t->ip;
4881
4882         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
4883
4884         /* Structs. */
4885         ALU_MI(t, ip, +);
4886
4887         /* Thread. */
4888         thread_ip_inc(p);
4889 }
4890
4891 static inline void
4892 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
4893 {
4894         struct thread *t = &p->threads[p->thread_id];
4895         struct instruction *ip = t->ip;
4896
4897         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
4898
4899         /* Structs. */
4900         ALU_HI(t, ip, +);
4901
4902         /* Thread. */
4903         thread_ip_inc(p);
4904 }
4905
4906 static inline void
4907 instr_alu_sub_exec(struct rte_swx_pipeline *p)
4908 {
4909         struct thread *t = &p->threads[p->thread_id];
4910         struct instruction *ip = t->ip;
4911
4912         TRACE("[Thread %2u] sub\n", p->thread_id);
4913
4914         /* Structs. */
4915         ALU(t, ip, -);
4916
4917         /* Thread. */
4918         thread_ip_inc(p);
4919 }
4920
4921 static inline void
4922 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
4923 {
4924         struct thread *t = &p->threads[p->thread_id];
4925         struct instruction *ip = t->ip;
4926
4927         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
4928
4929         /* Structs. */
4930         ALU_MH(t, ip, -);
4931
4932         /* Thread. */
4933         thread_ip_inc(p);
4934 }
4935
4936 static inline void
4937 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4938 {
4939         struct thread *t = &p->threads[p->thread_id];
4940         struct instruction *ip = t->ip;
4941
4942         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4943
4944         /* Structs. */
4945         ALU_HM(t, ip, -);
4946
4947         /* Thread. */
4948         thread_ip_inc(p);
4949 }
4950
4951 static inline void
4952 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4953 {
4954         struct thread *t = &p->threads[p->thread_id];
4955         struct instruction *ip = t->ip;
4956
4957         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4958
4959         /* Structs. */
4960         ALU_HH(t, ip, -);
4961
4962         /* Thread. */
4963         thread_ip_inc(p);
4964 }
4965
4966 static inline void
4967 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4968 {
4969         struct thread *t = &p->threads[p->thread_id];
4970         struct instruction *ip = t->ip;
4971
4972         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4973
4974         /* Structs. */
4975         ALU_MI(t, ip, -);
4976
4977         /* Thread. */
4978         thread_ip_inc(p);
4979 }
4980
4981 static inline void
4982 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4983 {
4984         struct thread *t = &p->threads[p->thread_id];
4985         struct instruction *ip = t->ip;
4986
4987         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4988
4989         /* Structs. */
4990         ALU_HI(t, ip, -);
4991
4992         /* Thread. */
4993         thread_ip_inc(p);
4994 }
4995
4996 static inline void
4997 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4998 {
4999         struct thread *t = &p->threads[p->thread_id];
5000         struct instruction *ip = t->ip;
5001
5002         TRACE("[Thread %2u] shl\n", p->thread_id);
5003
5004         /* Structs. */
5005         ALU(t, ip, <<);
5006
5007         /* Thread. */
5008         thread_ip_inc(p);
5009 }
5010
5011 static inline void
5012 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
5013 {
5014         struct thread *t = &p->threads[p->thread_id];
5015         struct instruction *ip = t->ip;
5016
5017         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
5018
5019         /* Structs. */
5020         ALU_MH(t, ip, <<);
5021
5022         /* Thread. */
5023         thread_ip_inc(p);
5024 }
5025
5026 static inline void
5027 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
5028 {
5029         struct thread *t = &p->threads[p->thread_id];
5030         struct instruction *ip = t->ip;
5031
5032         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
5033
5034         /* Structs. */
5035         ALU_HM(t, ip, <<);
5036
5037         /* Thread. */
5038         thread_ip_inc(p);
5039 }
5040
5041 static inline void
5042 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
5043 {
5044         struct thread *t = &p->threads[p->thread_id];
5045         struct instruction *ip = t->ip;
5046
5047         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
5048
5049         /* Structs. */
5050         ALU_HH(t, ip, <<);
5051
5052         /* Thread. */
5053         thread_ip_inc(p);
5054 }
5055
5056 static inline void
5057 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
5058 {
5059         struct thread *t = &p->threads[p->thread_id];
5060         struct instruction *ip = t->ip;
5061
5062         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
5063
5064         /* Structs. */
5065         ALU_MI(t, ip, <<);
5066
5067         /* Thread. */
5068         thread_ip_inc(p);
5069 }
5070
5071 static inline void
5072 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
5073 {
5074         struct thread *t = &p->threads[p->thread_id];
5075         struct instruction *ip = t->ip;
5076
5077         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
5078
5079         /* Structs. */
5080         ALU_HI(t, ip, <<);
5081
5082         /* Thread. */
5083         thread_ip_inc(p);
5084 }
5085
5086 static inline void
5087 instr_alu_shr_exec(struct rte_swx_pipeline *p)
5088 {
5089         struct thread *t = &p->threads[p->thread_id];
5090         struct instruction *ip = t->ip;
5091
5092         TRACE("[Thread %2u] shr\n", p->thread_id);
5093
5094         /* Structs. */
5095         ALU(t, ip, >>);
5096
5097         /* Thread. */
5098         thread_ip_inc(p);
5099 }
5100
5101 static inline void
5102 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
5103 {
5104         struct thread *t = &p->threads[p->thread_id];
5105         struct instruction *ip = t->ip;
5106
5107         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
5108
5109         /* Structs. */
5110         ALU_MH(t, ip, >>);
5111
5112         /* Thread. */
5113         thread_ip_inc(p);
5114 }
5115
5116 static inline void
5117 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
5118 {
5119         struct thread *t = &p->threads[p->thread_id];
5120         struct instruction *ip = t->ip;
5121
5122         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
5123
5124         /* Structs. */
5125         ALU_HM(t, ip, >>);
5126
5127         /* Thread. */
5128         thread_ip_inc(p);
5129 }
5130
5131 static inline void
5132 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
5133 {
5134         struct thread *t = &p->threads[p->thread_id];
5135         struct instruction *ip = t->ip;
5136
5137         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
5138
5139         /* Structs. */
5140         ALU_HH(t, ip, >>);
5141
5142         /* Thread. */
5143         thread_ip_inc(p);
5144 }
5145
5146 static inline void
5147 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
5148 {
5149         struct thread *t = &p->threads[p->thread_id];
5150         struct instruction *ip = t->ip;
5151
5152         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
5153
5154         /* Structs. */
5155         ALU_MI(t, ip, >>);
5156
5157         /* Thread. */
5158         thread_ip_inc(p);
5159 }
5160
5161 static inline void
5162 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
5163 {
5164         struct thread *t = &p->threads[p->thread_id];
5165         struct instruction *ip = t->ip;
5166
5167         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
5168
5169         /* Structs. */
5170         ALU_HI(t, ip, >>);
5171
5172         /* Thread. */
5173         thread_ip_inc(p);
5174 }
5175
5176 static inline void
5177 instr_alu_and_exec(struct rte_swx_pipeline *p)
5178 {
5179         struct thread *t = &p->threads[p->thread_id];
5180         struct instruction *ip = t->ip;
5181
5182         TRACE("[Thread %2u] and\n", p->thread_id);
5183
5184         /* Structs. */
5185         ALU(t, ip, &);
5186
5187         /* Thread. */
5188         thread_ip_inc(p);
5189 }
5190
5191 static inline void
5192 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
5193 {
5194         struct thread *t = &p->threads[p->thread_id];
5195         struct instruction *ip = t->ip;
5196
5197         TRACE("[Thread %2u] and (mh)\n", p->thread_id);
5198
5199         /* Structs. */
5200         ALU_MH(t, ip, &);
5201
5202         /* Thread. */
5203         thread_ip_inc(p);
5204 }
5205
5206 static inline void
5207 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
5208 {
5209         struct thread *t = &p->threads[p->thread_id];
5210         struct instruction *ip = t->ip;
5211
5212         TRACE("[Thread %2u] and (hm)\n", p->thread_id);
5213
5214         /* Structs. */
5215         ALU_HM_FAST(t, ip, &);
5216
5217         /* Thread. */
5218         thread_ip_inc(p);
5219 }
5220
5221 static inline void
5222 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
5223 {
5224         struct thread *t = &p->threads[p->thread_id];
5225         struct instruction *ip = t->ip;
5226
5227         TRACE("[Thread %2u] and (hh)\n", p->thread_id);
5228
5229         /* Structs. */
5230         ALU_HH_FAST(t, ip, &);
5231
5232         /* Thread. */
5233         thread_ip_inc(p);
5234 }
5235
5236 static inline void
5237 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
5238 {
5239         struct thread *t = &p->threads[p->thread_id];
5240         struct instruction *ip = t->ip;
5241
5242         TRACE("[Thread %2u] and (i)\n", p->thread_id);
5243
5244         /* Structs. */
5245         ALU_I(t, ip, &);
5246
5247         /* Thread. */
5248         thread_ip_inc(p);
5249 }
5250
5251 static inline void
5252 instr_alu_or_exec(struct rte_swx_pipeline *p)
5253 {
5254         struct thread *t = &p->threads[p->thread_id];
5255         struct instruction *ip = t->ip;
5256
5257         TRACE("[Thread %2u] or\n", p->thread_id);
5258
5259         /* Structs. */
5260         ALU(t, ip, |);
5261
5262         /* Thread. */
5263         thread_ip_inc(p);
5264 }
5265
5266 static inline void
5267 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
5268 {
5269         struct thread *t = &p->threads[p->thread_id];
5270         struct instruction *ip = t->ip;
5271
5272         TRACE("[Thread %2u] or (mh)\n", p->thread_id);
5273
5274         /* Structs. */
5275         ALU_MH(t, ip, |);
5276
5277         /* Thread. */
5278         thread_ip_inc(p);
5279 }
5280
5281 static inline void
5282 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
5283 {
5284         struct thread *t = &p->threads[p->thread_id];
5285         struct instruction *ip = t->ip;
5286
5287         TRACE("[Thread %2u] or (hm)\n", p->thread_id);
5288
5289         /* Structs. */
5290         ALU_HM_FAST(t, ip, |);
5291
5292         /* Thread. */
5293         thread_ip_inc(p);
5294 }
5295
5296 static inline void
5297 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
5298 {
5299         struct thread *t = &p->threads[p->thread_id];
5300         struct instruction *ip = t->ip;
5301
5302         TRACE("[Thread %2u] or (hh)\n", p->thread_id);
5303
5304         /* Structs. */
5305         ALU_HH_FAST(t, ip, |);
5306
5307         /* Thread. */
5308         thread_ip_inc(p);
5309 }
5310
5311 static inline void
5312 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
5313 {
5314         struct thread *t = &p->threads[p->thread_id];
5315         struct instruction *ip = t->ip;
5316
5317         TRACE("[Thread %2u] or (i)\n", p->thread_id);
5318
5319         /* Structs. */
5320         ALU_I(t, ip, |);
5321
5322         /* Thread. */
5323         thread_ip_inc(p);
5324 }
5325
5326 static inline void
5327 instr_alu_xor_exec(struct rte_swx_pipeline *p)
5328 {
5329         struct thread *t = &p->threads[p->thread_id];
5330         struct instruction *ip = t->ip;
5331
5332         TRACE("[Thread %2u] xor\n", p->thread_id);
5333
5334         /* Structs. */
5335         ALU(t, ip, ^);
5336
5337         /* Thread. */
5338         thread_ip_inc(p);
5339 }
5340
5341 static inline void
5342 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
5343 {
5344         struct thread *t = &p->threads[p->thread_id];
5345         struct instruction *ip = t->ip;
5346
5347         TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
5348
5349         /* Structs. */
5350         ALU_MH(t, ip, ^);
5351
5352         /* Thread. */
5353         thread_ip_inc(p);
5354 }
5355
5356 static inline void
5357 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
5358 {
5359         struct thread *t = &p->threads[p->thread_id];
5360         struct instruction *ip = t->ip;
5361
5362         TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
5363
5364         /* Structs. */
5365         ALU_HM_FAST(t, ip, ^);
5366
5367         /* Thread. */
5368         thread_ip_inc(p);
5369 }
5370
5371 static inline void
5372 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
5373 {
5374         struct thread *t = &p->threads[p->thread_id];
5375         struct instruction *ip = t->ip;
5376
5377         TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
5378
5379         /* Structs. */
5380         ALU_HH_FAST(t, ip, ^);
5381
5382         /* Thread. */
5383         thread_ip_inc(p);
5384 }
5385
5386 static inline void
5387 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
5388 {
5389         struct thread *t = &p->threads[p->thread_id];
5390         struct instruction *ip = t->ip;
5391
5392         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
5393
5394         /* Structs. */
5395         ALU_I(t, ip, ^);
5396
5397         /* Thread. */
5398         thread_ip_inc(p);
5399 }
5400
5401 static inline void
5402 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
5403 {
5404         struct thread *t = &p->threads[p->thread_id];
5405         struct instruction *ip = t->ip;
5406         uint8_t *dst_struct, *src_struct;
5407         uint16_t *dst16_ptr, dst;
5408         uint64_t *src64_ptr, src64, src64_mask, src;
5409         uint64_t r;
5410
5411         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
5412
5413         /* Structs. */
5414         dst_struct = t->structs[ip->alu.dst.struct_id];
5415         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5416         dst = *dst16_ptr;
5417
5418         src_struct = t->structs[ip->alu.src.struct_id];
5419         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
5420         src64 = *src64_ptr;
5421         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
5422         src = src64 & src64_mask;
5423
5424         r = dst;
5425         r = ~r & 0xFFFF;
5426
5427         /* The first input (r) is a 16-bit number. The second and the third
5428          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
5429          * three numbers (output r) is a 34-bit number.
5430          */
5431         r += (src >> 32) + (src & 0xFFFFFFFF);
5432
5433         /* The first input is a 16-bit number. The second input is an 18-bit
5434          * number. In the worst case scenario, the sum of the two numbers is a
5435          * 19-bit number.
5436          */
5437         r = (r & 0xFFFF) + (r >> 16);
5438
5439         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5440          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
5441          */
5442         r = (r & 0xFFFF) + (r >> 16);
5443
5444         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5445          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5446          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
5447          * therefore the output r is always a 16-bit number.
5448          */
5449         r = (r & 0xFFFF) + (r >> 16);
5450
5451         r = ~r & 0xFFFF;
5452         r = r ? r : 0xFFFF;
5453
5454         *dst16_ptr = (uint16_t)r;
5455
5456         /* Thread. */
5457         thread_ip_inc(p);
5458 }
5459
5460 static inline void
5461 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
5462 {
5463         struct thread *t = &p->threads[p->thread_id];
5464         struct instruction *ip = t->ip;
5465         uint8_t *dst_struct, *src_struct;
5466         uint16_t *dst16_ptr, dst;
5467         uint64_t *src64_ptr, src64, src64_mask, src;
5468         uint64_t r;
5469
5470         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
5471
5472         /* Structs. */
5473         dst_struct = t->structs[ip->alu.dst.struct_id];
5474         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5475         dst = *dst16_ptr;
5476
5477         src_struct = t->structs[ip->alu.src.struct_id];
5478         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
5479         src64 = *src64_ptr;
5480         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
5481         src = src64 & src64_mask;
5482
5483         r = dst;
5484         r = ~r & 0xFFFF;
5485
5486         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
5487          * the following sequence of operations in 2's complement arithmetic:
5488          *    a '- b = (a - b) % 0xFFFF.
5489          *
5490          * In order to prevent an underflow for the below subtraction, in which
5491          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
5492          * minuend), we first add a multiple of the 0xFFFF modulus to the
5493          * minuend. The number we add to the minuend needs to be a 34-bit number
5494          * or higher, so for readability reasons we picked the 36-bit multiple.
5495          * We are effectively turning the 16-bit minuend into a 36-bit number:
5496          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
5497          */
5498         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
5499
5500         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
5501          * result (the output r) is a 36-bit number.
5502          */
5503         r -= (src >> 32) + (src & 0xFFFFFFFF);
5504
5505         /* The first input is a 16-bit number. The second input is a 20-bit
5506          * number. Their sum is a 21-bit number.
5507          */
5508         r = (r & 0xFFFF) + (r >> 16);
5509
5510         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5511          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
5512          */
5513         r = (r & 0xFFFF) + (r >> 16);
5514
5515         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5516          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5517          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5518          * generated, therefore the output r is always a 16-bit number.
5519          */
5520         r = (r & 0xFFFF) + (r >> 16);
5521
5522         r = ~r & 0xFFFF;
5523         r = r ? r : 0xFFFF;
5524
5525         *dst16_ptr = (uint16_t)r;
5526
5527         /* Thread. */
5528         thread_ip_inc(p);
5529 }
5530
5531 static inline void
5532 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
5533 {
5534         struct thread *t = &p->threads[p->thread_id];
5535         struct instruction *ip = t->ip;
5536         uint8_t *dst_struct, *src_struct;
5537         uint16_t *dst16_ptr;
5538         uint32_t *src32_ptr;
5539         uint64_t r0, r1;
5540
5541         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
5542
5543         /* Structs. */
5544         dst_struct = t->structs[ip->alu.dst.struct_id];
5545         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5546
5547         src_struct = t->structs[ip->alu.src.struct_id];
5548         src32_ptr = (uint32_t *)&src_struct[0];
5549
5550         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
5551         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
5552         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
5553         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
5554         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
5555
5556         /* The first input is a 16-bit number. The second input is a 19-bit
5557          * number. Their sum is a 20-bit number.
5558          */
5559         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5560
5561         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5562          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
5563          */
5564         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5565
5566         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5567          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5568          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
5569          * generated, therefore the output r is always a 16-bit number.
5570          */
5571         r0 = (r0 & 0xFFFF) + (r0 >> 16);
5572
5573         r0 = ~r0 & 0xFFFF;
5574         r0 = r0 ? r0 : 0xFFFF;
5575
5576         *dst16_ptr = (uint16_t)r0;
5577
5578         /* Thread. */
5579         thread_ip_inc(p);
5580 }
5581
5582 static inline void
5583 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
5584 {
5585         struct thread *t = &p->threads[p->thread_id];
5586         struct instruction *ip = t->ip;
5587         uint8_t *dst_struct, *src_struct;
5588         uint16_t *dst16_ptr;
5589         uint32_t *src32_ptr;
5590         uint64_t r = 0;
5591         uint32_t i;
5592
5593         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
5594
5595         /* Structs. */
5596         dst_struct = t->structs[ip->alu.dst.struct_id];
5597         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5598
5599         src_struct = t->structs[ip->alu.src.struct_id];
5600         src32_ptr = (uint32_t *)&src_struct[0];
5601
5602         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
5603          * Therefore, in the worst case scenario, a 35-bit number is added to a
5604          * 16-bit number (the input r), so the output r is 36-bit number.
5605          */
5606         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
5607                 r += *src32_ptr;
5608
5609         /* The first input is a 16-bit number. The second input is a 20-bit
5610          * number. Their sum is a 21-bit number.
5611          */
5612         r = (r & 0xFFFF) + (r >> 16);
5613
5614         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5615          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
5616          */
5617         r = (r & 0xFFFF) + (r >> 16);
5618
5619         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5620          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5621          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5622          * generated, therefore the output r is always a 16-bit number.
5623          */
5624         r = (r & 0xFFFF) + (r >> 16);
5625
5626         r = ~r & 0xFFFF;
5627         r = r ? r : 0xFFFF;
5628
5629         *dst16_ptr = (uint16_t)r;
5630
5631         /* Thread. */
5632         thread_ip_inc(p);
5633 }
5634
5635 /*
5636  * Register array.
5637  */
5638 static struct regarray *
5639 regarray_find(struct rte_swx_pipeline *p, const char *name);
5640
5641 static int
5642 instr_regprefetch_translate(struct rte_swx_pipeline *p,
5643                       struct action *action,
5644                       char **tokens,
5645                       int n_tokens,
5646                       struct instruction *instr,
5647                       struct instruction_data *data __rte_unused)
5648 {
5649         char *regarray = tokens[1], *idx = tokens[2];
5650         struct regarray *r;
5651         struct field *fidx;
5652         uint32_t idx_struct_id, idx_val;
5653
5654         CHECK(n_tokens == 3, EINVAL);
5655
5656         r = regarray_find(p, regarray);
5657         CHECK(r, EINVAL);
5658
5659         /* REGPREFETCH_RH, REGPREFETCH_RM. */
5660         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5661         if (fidx) {
5662                 CHECK(!fidx->var_size, EINVAL);
5663
5664                 instr->type = INSTR_REGPREFETCH_RM;
5665                 if (idx[0] == 'h')
5666                         instr->type = INSTR_REGPREFETCH_RH;
5667
5668                 instr->regarray.regarray_id = r->id;
5669                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5670                 instr->regarray.idx.n_bits = fidx->n_bits;
5671                 instr->regarray.idx.offset = fidx->offset / 8;
5672                 instr->regarray.dstsrc_val = 0; /* Unused. */
5673                 return 0;
5674         }
5675
5676         /* REGPREFETCH_RI. */
5677         idx_val = strtoul(idx, &idx, 0);
5678         CHECK(!idx[0], EINVAL);
5679
5680         instr->type = INSTR_REGPREFETCH_RI;
5681         instr->regarray.regarray_id = r->id;
5682         instr->regarray.idx_val = idx_val;
5683         instr->regarray.dstsrc_val = 0; /* Unused. */
5684         return 0;
5685 }
5686
5687 static int
5688 instr_regrd_translate(struct rte_swx_pipeline *p,
5689                       struct action *action,
5690                       char **tokens,
5691                       int n_tokens,
5692                       struct instruction *instr,
5693                       struct instruction_data *data __rte_unused)
5694 {
5695         char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
5696         struct regarray *r;
5697         struct field *fdst, *fidx;
5698         uint32_t dst_struct_id, idx_struct_id, idx_val;
5699
5700         CHECK(n_tokens == 4, EINVAL);
5701
5702         r = regarray_find(p, regarray);
5703         CHECK(r, EINVAL);
5704
5705         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
5706         CHECK(fdst, EINVAL);
5707         CHECK(!fdst->var_size, EINVAL);
5708
5709         /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
5710         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5711         if (fidx) {
5712                 CHECK(!fidx->var_size, EINVAL);
5713
5714                 instr->type = INSTR_REGRD_MRM;
5715                 if (dst[0] == 'h' && idx[0] != 'h')
5716                         instr->type = INSTR_REGRD_HRM;
5717                 if (dst[0] != 'h' && idx[0] == 'h')
5718                         instr->type = INSTR_REGRD_MRH;
5719                 if (dst[0] == 'h' && idx[0] == 'h')
5720                         instr->type = INSTR_REGRD_HRH;
5721
5722                 instr->regarray.regarray_id = r->id;
5723                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5724                 instr->regarray.idx.n_bits = fidx->n_bits;
5725                 instr->regarray.idx.offset = fidx->offset / 8;
5726                 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5727                 instr->regarray.dstsrc.n_bits = fdst->n_bits;
5728                 instr->regarray.dstsrc.offset = fdst->offset / 8;
5729                 return 0;
5730         }
5731
5732         /* REGRD_MRI, REGRD_HRI. */
5733         idx_val = strtoul(idx, &idx, 0);
5734         CHECK(!idx[0], EINVAL);
5735
5736         instr->type = INSTR_REGRD_MRI;
5737         if (dst[0] == 'h')
5738                 instr->type = INSTR_REGRD_HRI;
5739
5740         instr->regarray.regarray_id = r->id;
5741         instr->regarray.idx_val = idx_val;
5742         instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5743         instr->regarray.dstsrc.n_bits = fdst->n_bits;
5744         instr->regarray.dstsrc.offset = fdst->offset / 8;
5745         return 0;
5746 }
5747
5748 static int
5749 instr_regwr_translate(struct rte_swx_pipeline *p,
5750                       struct action *action,
5751                       char **tokens,
5752                       int n_tokens,
5753                       struct instruction *instr,
5754                       struct instruction_data *data __rte_unused)
5755 {
5756         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5757         struct regarray *r;
5758         struct field *fidx, *fsrc;
5759         uint64_t src_val;
5760         uint32_t idx_struct_id, idx_val, src_struct_id;
5761
5762         CHECK(n_tokens == 4, EINVAL);
5763
5764         r = regarray_find(p, regarray);
5765         CHECK(r, EINVAL);
5766
5767         /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
5768         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5769         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5770         if (fidx && fsrc) {
5771                 CHECK(!fidx->var_size, EINVAL);
5772                 CHECK(!fsrc->var_size, EINVAL);
5773
5774                 instr->type = INSTR_REGWR_RMM;
5775                 if (idx[0] == 'h' && src[0] != 'h')
5776                         instr->type = INSTR_REGWR_RHM;
5777                 if (idx[0] != 'h' && src[0] == 'h')
5778                         instr->type = INSTR_REGWR_RMH;
5779                 if (idx[0] == 'h' && src[0] == 'h')
5780                         instr->type = INSTR_REGWR_RHH;
5781
5782                 instr->regarray.regarray_id = r->id;
5783                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5784                 instr->regarray.idx.n_bits = fidx->n_bits;
5785                 instr->regarray.idx.offset = fidx->offset / 8;
5786                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5787                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5788                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5789                 return 0;
5790         }
5791
5792         /* REGWR_RHI, REGWR_RMI. */
5793         if (fidx && !fsrc) {
5794                 CHECK(!fidx->var_size, EINVAL);
5795
5796                 src_val = strtoull(src, &src, 0);
5797                 CHECK(!src[0], EINVAL);
5798
5799                 instr->type = INSTR_REGWR_RMI;
5800                 if (idx[0] == 'h')
5801                         instr->type = INSTR_REGWR_RHI;
5802
5803                 instr->regarray.regarray_id = r->id;
5804                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5805                 instr->regarray.idx.n_bits = fidx->n_bits;
5806                 instr->regarray.idx.offset = fidx->offset / 8;
5807                 instr->regarray.dstsrc_val = src_val;
5808                 return 0;
5809         }
5810
5811         /* REGWR_RIH, REGWR_RIM. */
5812         if (!fidx && fsrc) {
5813                 idx_val = strtoul(idx, &idx, 0);
5814                 CHECK(!idx[0], EINVAL);
5815
5816                 CHECK(!fsrc->var_size, EINVAL);
5817
5818                 instr->type = INSTR_REGWR_RIM;
5819                 if (src[0] == 'h')
5820                         instr->type = INSTR_REGWR_RIH;
5821
5822                 instr->regarray.regarray_id = r->id;
5823                 instr->regarray.idx_val = idx_val;
5824                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5825                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5826                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5827                 return 0;
5828         }
5829
5830         /* REGWR_RII. */
5831         src_val = strtoull(src, &src, 0);
5832         CHECK(!src[0], EINVAL);
5833
5834         idx_val = strtoul(idx, &idx, 0);
5835         CHECK(!idx[0], EINVAL);
5836
5837         instr->type = INSTR_REGWR_RII;
5838         instr->regarray.idx_val = idx_val;
5839         instr->regarray.dstsrc_val = src_val;
5840
5841         return 0;
5842 }
5843
5844 static int
5845 instr_regadd_translate(struct rte_swx_pipeline *p,
5846                        struct action *action,
5847                        char **tokens,
5848                        int n_tokens,
5849                        struct instruction *instr,
5850                        struct instruction_data *data __rte_unused)
5851 {
5852         char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5853         struct regarray *r;
5854         struct field *fidx, *fsrc;
5855         uint64_t src_val;
5856         uint32_t idx_struct_id, idx_val, src_struct_id;
5857
5858         CHECK(n_tokens == 4, EINVAL);
5859
5860         r = regarray_find(p, regarray);
5861         CHECK(r, EINVAL);
5862
5863         /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
5864         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5865         fsrc = struct_field_parse(p, action, src, &src_struct_id);
5866         if (fidx && fsrc) {
5867                 CHECK(!fidx->var_size, EINVAL);
5868                 CHECK(!fsrc->var_size, EINVAL);
5869
5870                 instr->type = INSTR_REGADD_RMM;
5871                 if (idx[0] == 'h' && src[0] != 'h')
5872                         instr->type = INSTR_REGADD_RHM;
5873                 if (idx[0] != 'h' && src[0] == 'h')
5874                         instr->type = INSTR_REGADD_RMH;
5875                 if (idx[0] == 'h' && src[0] == 'h')
5876                         instr->type = INSTR_REGADD_RHH;
5877
5878                 instr->regarray.regarray_id = r->id;
5879                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5880                 instr->regarray.idx.n_bits = fidx->n_bits;
5881                 instr->regarray.idx.offset = fidx->offset / 8;
5882                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5883                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5884                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5885                 return 0;
5886         }
5887
5888         /* REGADD_RHI, REGADD_RMI. */
5889         if (fidx && !fsrc) {
5890                 CHECK(!fidx->var_size, EINVAL);
5891
5892                 src_val = strtoull(src, &src, 0);
5893                 CHECK(!src[0], EINVAL);
5894
5895                 instr->type = INSTR_REGADD_RMI;
5896                 if (idx[0] == 'h')
5897                         instr->type = INSTR_REGADD_RHI;
5898
5899                 instr->regarray.regarray_id = r->id;
5900                 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5901                 instr->regarray.idx.n_bits = fidx->n_bits;
5902                 instr->regarray.idx.offset = fidx->offset / 8;
5903                 instr->regarray.dstsrc_val = src_val;
5904                 return 0;
5905         }
5906
5907         /* REGADD_RIH, REGADD_RIM. */
5908         if (!fidx && fsrc) {
5909                 idx_val = strtoul(idx, &idx, 0);
5910                 CHECK(!idx[0], EINVAL);
5911
5912                 CHECK(!fsrc->var_size, EINVAL);
5913
5914                 instr->type = INSTR_REGADD_RIM;
5915                 if (src[0] == 'h')
5916                         instr->type = INSTR_REGADD_RIH;
5917
5918                 instr->regarray.regarray_id = r->id;
5919                 instr->regarray.idx_val = idx_val;
5920                 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5921                 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5922                 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5923                 return 0;
5924         }
5925
5926         /* REGADD_RII. */
5927         src_val = strtoull(src, &src, 0);
5928         CHECK(!src[0], EINVAL);
5929
5930         idx_val = strtoul(idx, &idx, 0);
5931         CHECK(!idx[0], EINVAL);
5932
5933         instr->type = INSTR_REGADD_RII;
5934         instr->regarray.idx_val = idx_val;
5935         instr->regarray.dstsrc_val = src_val;
5936         return 0;
5937 }
5938
5939 static inline uint64_t *
5940 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
5941 {
5942         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5943         return r->regarray;
5944 }
5945
5946 static inline uint64_t
5947 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5948 {
5949         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5950
5951         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5952         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5953         uint64_t idx64 = *idx64_ptr;
5954         uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
5955         uint64_t idx = idx64 & idx64_mask & r->size_mask;
5956
5957         return idx;
5958 }
5959
5960 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5961
5962 static inline uint64_t
5963 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5964 {
5965         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5966
5967         uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5968         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5969         uint64_t idx64 = *idx64_ptr;
5970         uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
5971
5972         return idx;
5973 }
5974
5975 #else
5976
5977 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
5978
5979 #endif
5980
5981 static inline uint64_t
5982 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5983 {
5984         struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5985
5986         uint64_t idx = ip->regarray.idx_val & r->size_mask;
5987
5988         return idx;
5989 }
5990
5991 static inline uint64_t
5992 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
5993 {
5994         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5995         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5996         uint64_t src64 = *src64_ptr;
5997         uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5998         uint64_t src = src64 & src64_mask;
5999
6000         return src;
6001 }
6002
6003 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6004
6005 static inline uint64_t
6006 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
6007 {
6008         uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
6009         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
6010         uint64_t src64 = *src64_ptr;
6011         uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
6012
6013         return src;
6014 }
6015
6016 #else
6017
6018 #define instr_regarray_src_nbo instr_regarray_src_hbo
6019
6020 #endif
6021
6022 static inline void
6023 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
6024 {
6025         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
6026         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
6027         uint64_t dst64 = *dst64_ptr;
6028         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
6029
6030         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6031
6032 }
6033
6034 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6035
6036 static inline void
6037 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
6038 {
6039         uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
6040         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
6041         uint64_t dst64 = *dst64_ptr;
6042         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
6043
6044         src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
6045         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6046 }
6047
6048 #else
6049
6050 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
6051
6052 #endif
6053
6054 static inline void
6055 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
6056 {
6057         struct thread *t = &p->threads[p->thread_id];
6058         struct instruction *ip = t->ip;
6059         uint64_t *regarray, idx;
6060
6061         TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
6062
6063         /* Structs. */
6064         regarray = instr_regarray_regarray(p, ip);
6065         idx = instr_regarray_idx_nbo(p, t, ip);
6066         rte_prefetch0(&regarray[idx]);
6067
6068         /* Thread. */
6069         thread_ip_inc(p);
6070 }
6071
6072 static inline void
6073 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
6074 {
6075         struct thread *t = &p->threads[p->thread_id];
6076         struct instruction *ip = t->ip;
6077         uint64_t *regarray, idx;
6078
6079         TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
6080
6081         /* Structs. */
6082         regarray = instr_regarray_regarray(p, ip);
6083         idx = instr_regarray_idx_hbo(p, t, ip);
6084         rte_prefetch0(&regarray[idx]);
6085
6086         /* Thread. */
6087         thread_ip_inc(p);
6088 }
6089
6090 static inline void
6091 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
6092 {
6093         struct thread *t = &p->threads[p->thread_id];
6094         struct instruction *ip = t->ip;
6095         uint64_t *regarray, idx;
6096
6097         TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
6098
6099         /* Structs. */
6100         regarray = instr_regarray_regarray(p, ip);
6101         idx = instr_regarray_idx_imm(p, ip);
6102         rte_prefetch0(&regarray[idx]);
6103
6104         /* Thread. */
6105         thread_ip_inc(p);
6106 }
6107
6108 static inline void
6109 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
6110 {
6111         struct thread *t = &p->threads[p->thread_id];
6112         struct instruction *ip = t->ip;
6113         uint64_t *regarray, idx;
6114
6115         TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
6116
6117         /* Structs. */
6118         regarray = instr_regarray_regarray(p, ip);
6119         idx = instr_regarray_idx_nbo(p, t, ip);
6120         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
6121
6122         /* Thread. */
6123         thread_ip_inc(p);
6124 }
6125
6126 static inline void
6127 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
6128 {
6129         struct thread *t = &p->threads[p->thread_id];
6130         struct instruction *ip = t->ip;
6131         uint64_t *regarray, idx;
6132
6133         TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
6134
6135         /* Structs. */
6136         regarray = instr_regarray_regarray(p, ip);
6137         idx = instr_regarray_idx_hbo(p, t, ip);
6138         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
6139
6140         /* Thread. */
6141         thread_ip_inc(p);
6142 }
6143
6144 static inline void
6145 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
6146 {
6147         struct thread *t = &p->threads[p->thread_id];
6148         struct instruction *ip = t->ip;
6149         uint64_t *regarray, idx;
6150
6151         TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
6152
6153         /* Structs. */
6154         regarray = instr_regarray_regarray(p, ip);
6155         idx = instr_regarray_idx_nbo(p, t, ip);
6156         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
6157
6158         /* Thread. */
6159         thread_ip_inc(p);
6160 }
6161
6162 static inline void
6163 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
6164 {
6165         struct thread *t = &p->threads[p->thread_id];
6166         struct instruction *ip = t->ip;
6167         uint64_t *regarray, idx;
6168
6169         /* Structs. */
6170         regarray = instr_regarray_regarray(p, ip);
6171         idx = instr_regarray_idx_hbo(p, t, ip);
6172         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
6173
6174         /* Thread. */
6175         thread_ip_inc(p);
6176 }
6177
6178 static inline void
6179 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
6180 {
6181         struct thread *t = &p->threads[p->thread_id];
6182         struct instruction *ip = t->ip;
6183         uint64_t *regarray, idx;
6184
6185         TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
6186
6187         /* Structs. */
6188         regarray = instr_regarray_regarray(p, ip);
6189         idx = instr_regarray_idx_imm(p, ip);
6190         instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
6191
6192         /* Thread. */
6193         thread_ip_inc(p);
6194 }
6195
6196 static inline void
6197 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
6198 {
6199         struct thread *t = &p->threads[p->thread_id];
6200         struct instruction *ip = t->ip;
6201         uint64_t *regarray, idx;
6202
6203         TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
6204
6205         /* Structs. */
6206         regarray = instr_regarray_regarray(p, ip);
6207         idx = instr_regarray_idx_imm(p, ip);
6208         instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
6209
6210         /* Thread. */
6211         thread_ip_inc(p);
6212 }
6213
6214 static inline void
6215 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
6216 {
6217         struct thread *t = &p->threads[p->thread_id];
6218         struct instruction *ip = t->ip;
6219         uint64_t *regarray, idx, src;
6220
6221         TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
6222
6223         /* Structs. */
6224         regarray = instr_regarray_regarray(p, ip);
6225         idx = instr_regarray_idx_nbo(p, t, ip);
6226         src = instr_regarray_src_nbo(t, ip);
6227         regarray[idx] = src;
6228
6229         /* Thread. */
6230         thread_ip_inc(p);
6231 }
6232
6233 static inline void
6234 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
6235 {
6236         struct thread *t = &p->threads[p->thread_id];
6237         struct instruction *ip = t->ip;
6238         uint64_t *regarray, idx, src;
6239
6240         TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
6241
6242         /* Structs. */
6243         regarray = instr_regarray_regarray(p, ip);
6244         idx = instr_regarray_idx_nbo(p, t, ip);
6245         src = instr_regarray_src_hbo(t, ip);
6246         regarray[idx] = src;
6247
6248         /* Thread. */
6249         thread_ip_inc(p);
6250 }
6251
6252 static inline void
6253 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
6254 {
6255         struct thread *t = &p->threads[p->thread_id];
6256         struct instruction *ip = t->ip;
6257         uint64_t *regarray, idx, src;
6258
6259         TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
6260
6261         /* Structs. */
6262         regarray = instr_regarray_regarray(p, ip);
6263         idx = instr_regarray_idx_hbo(p, t, ip);
6264         src = instr_regarray_src_nbo(t, ip);
6265         regarray[idx] = src;
6266
6267         /* Thread. */
6268         thread_ip_inc(p);
6269 }
6270
6271 static inline void
6272 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
6273 {
6274         struct thread *t = &p->threads[p->thread_id];
6275         struct instruction *ip = t->ip;
6276         uint64_t *regarray, idx, src;
6277
6278         TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
6279
6280         /* Structs. */
6281         regarray = instr_regarray_regarray(p, ip);
6282         idx = instr_regarray_idx_hbo(p, t, ip);
6283         src = instr_regarray_src_hbo(t, ip);
6284         regarray[idx] = src;
6285
6286         /* Thread. */
6287         thread_ip_inc(p);
6288 }
6289
6290 static inline void
6291 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
6292 {
6293         struct thread *t = &p->threads[p->thread_id];
6294         struct instruction *ip = t->ip;
6295         uint64_t *regarray, idx, src;
6296
6297         TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
6298
6299         /* Structs. */
6300         regarray = instr_regarray_regarray(p, ip);
6301         idx = instr_regarray_idx_nbo(p, t, ip);
6302         src = ip->regarray.dstsrc_val;
6303         regarray[idx] = src;
6304
6305         /* Thread. */
6306         thread_ip_inc(p);
6307 }
6308
6309 static inline void
6310 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
6311 {
6312         struct thread *t = &p->threads[p->thread_id];
6313         struct instruction *ip = t->ip;
6314         uint64_t *regarray, idx, src;
6315
6316         TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
6317
6318         /* Structs. */
6319         regarray = instr_regarray_regarray(p, ip);
6320         idx = instr_regarray_idx_hbo(p, t, ip);
6321         src = ip->regarray.dstsrc_val;
6322         regarray[idx] = src;
6323
6324         /* Thread. */
6325         thread_ip_inc(p);
6326 }
6327
6328 static inline void
6329 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
6330 {
6331         struct thread *t = &p->threads[p->thread_id];
6332         struct instruction *ip = t->ip;
6333         uint64_t *regarray, idx, src;
6334
6335         TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
6336
6337         /* Structs. */
6338         regarray = instr_regarray_regarray(p, ip);
6339         idx = instr_regarray_idx_imm(p, ip);
6340         src = instr_regarray_src_nbo(t, ip);
6341         regarray[idx] = src;
6342
6343         /* Thread. */
6344         thread_ip_inc(p);
6345 }
6346
6347 static inline void
6348 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
6349 {
6350         struct thread *t = &p->threads[p->thread_id];
6351         struct instruction *ip = t->ip;
6352         uint64_t *regarray, idx, src;
6353
6354         TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
6355
6356         /* Structs. */
6357         regarray = instr_regarray_regarray(p, ip);
6358         idx = instr_regarray_idx_imm(p, ip);
6359         src = instr_regarray_src_hbo(t, ip);
6360         regarray[idx] = src;
6361
6362         /* Thread. */
6363         thread_ip_inc(p);
6364 }
6365
6366 static inline void
6367 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
6368 {
6369         struct thread *t = &p->threads[p->thread_id];
6370         struct instruction *ip = t->ip;
6371         uint64_t *regarray, idx, src;
6372
6373         TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
6374
6375         /* Structs. */
6376         regarray = instr_regarray_regarray(p, ip);
6377         idx = instr_regarray_idx_imm(p, ip);
6378         src = ip->regarray.dstsrc_val;
6379         regarray[idx] = src;
6380
6381         /* Thread. */
6382         thread_ip_inc(p);
6383 }
6384
6385 static inline void
6386 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
6387 {
6388         struct thread *t = &p->threads[p->thread_id];
6389         struct instruction *ip = t->ip;
6390         uint64_t *regarray, idx, src;
6391
6392         TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
6393
6394         /* Structs. */
6395         regarray = instr_regarray_regarray(p, ip);
6396         idx = instr_regarray_idx_nbo(p, t, ip);
6397         src = instr_regarray_src_nbo(t, ip);
6398         regarray[idx] += src;
6399
6400         /* Thread. */
6401         thread_ip_inc(p);
6402 }
6403
6404 static inline void
6405 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
6406 {
6407         struct thread *t = &p->threads[p->thread_id];
6408         struct instruction *ip = t->ip;
6409         uint64_t *regarray, idx, src;
6410
6411         TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
6412
6413         /* Structs. */
6414         regarray = instr_regarray_regarray(p, ip);
6415         idx = instr_regarray_idx_nbo(p, t, ip);
6416         src = instr_regarray_src_hbo(t, ip);
6417         regarray[idx] += src;
6418
6419         /* Thread. */
6420         thread_ip_inc(p);
6421 }
6422
6423 static inline void
6424 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
6425 {
6426         struct thread *t = &p->threads[p->thread_id];
6427         struct instruction *ip = t->ip;
6428         uint64_t *regarray, idx, src;
6429
6430         TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
6431
6432         /* Structs. */
6433         regarray = instr_regarray_regarray(p, ip);
6434         idx = instr_regarray_idx_hbo(p, t, ip);
6435         src = instr_regarray_src_nbo(t, ip);
6436         regarray[idx] += src;
6437
6438         /* Thread. */
6439         thread_ip_inc(p);
6440 }
6441
6442 static inline void
6443 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
6444 {
6445         struct thread *t = &p->threads[p->thread_id];
6446         struct instruction *ip = t->ip;
6447         uint64_t *regarray, idx, src;
6448
6449         TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
6450
6451         /* Structs. */
6452         regarray = instr_regarray_regarray(p, ip);
6453         idx = instr_regarray_idx_hbo(p, t, ip);
6454         src = instr_regarray_src_hbo(t, ip);
6455         regarray[idx] += src;
6456
6457         /* Thread. */
6458         thread_ip_inc(p);
6459 }
6460
6461 static inline void
6462 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
6463 {
6464         struct thread *t = &p->threads[p->thread_id];
6465         struct instruction *ip = t->ip;
6466         uint64_t *regarray, idx, src;
6467
6468         TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
6469
6470         /* Structs. */
6471         regarray = instr_regarray_regarray(p, ip);
6472         idx = instr_regarray_idx_nbo(p, t, ip);
6473         src = ip->regarray.dstsrc_val;
6474         regarray[idx] += src;
6475
6476         /* Thread. */
6477         thread_ip_inc(p);
6478 }
6479
6480 static inline void
6481 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
6482 {
6483         struct thread *t = &p->threads[p->thread_id];
6484         struct instruction *ip = t->ip;
6485         uint64_t *regarray, idx, src;
6486
6487         TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
6488
6489         /* Structs. */
6490         regarray = instr_regarray_regarray(p, ip);
6491         idx = instr_regarray_idx_hbo(p, t, ip);
6492         src = ip->regarray.dstsrc_val;
6493         regarray[idx] += src;
6494
6495         /* Thread. */
6496         thread_ip_inc(p);
6497 }
6498
6499 static inline void
6500 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
6501 {
6502         struct thread *t = &p->threads[p->thread_id];
6503         struct instruction *ip = t->ip;
6504         uint64_t *regarray, idx, src;
6505
6506         TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
6507
6508         /* Structs. */
6509         regarray = instr_regarray_regarray(p, ip);
6510         idx = instr_regarray_idx_imm(p, ip);
6511         src = instr_regarray_src_nbo(t, ip);
6512         regarray[idx] += src;
6513
6514         /* Thread. */
6515         thread_ip_inc(p);
6516 }
6517
6518 static inline void
6519 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
6520 {
6521         struct thread *t = &p->threads[p->thread_id];
6522         struct instruction *ip = t->ip;
6523         uint64_t *regarray, idx, src;
6524
6525         TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
6526
6527         /* Structs. */
6528         regarray = instr_regarray_regarray(p, ip);
6529         idx = instr_regarray_idx_imm(p, ip);
6530         src = instr_regarray_src_hbo(t, ip);
6531         regarray[idx] += src;
6532
6533         /* Thread. */
6534         thread_ip_inc(p);
6535 }
6536
6537 static inline void
6538 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
6539 {
6540         struct thread *t = &p->threads[p->thread_id];
6541         struct instruction *ip = t->ip;
6542         uint64_t *regarray, idx, src;
6543
6544         TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
6545
6546         /* Structs. */
6547         regarray = instr_regarray_regarray(p, ip);
6548         idx = instr_regarray_idx_imm(p, ip);
6549         src = ip->regarray.dstsrc_val;
6550         regarray[idx] += src;
6551
6552         /* Thread. */
6553         thread_ip_inc(p);
6554 }
6555
6556 /*
6557  * metarray.
6558  */
6559 static struct metarray *
6560 metarray_find(struct rte_swx_pipeline *p, const char *name);
6561
6562 static int
6563 instr_metprefetch_translate(struct rte_swx_pipeline *p,
6564                             struct action *action,
6565                             char **tokens,
6566                             int n_tokens,
6567                             struct instruction *instr,
6568                             struct instruction_data *data __rte_unused)
6569 {
6570         char *metarray = tokens[1], *idx = tokens[2];
6571         struct metarray *m;
6572         struct field *fidx;
6573         uint32_t idx_struct_id, idx_val;
6574
6575         CHECK(n_tokens == 3, EINVAL);
6576
6577         m = metarray_find(p, metarray);
6578         CHECK(m, EINVAL);
6579
6580         /* METPREFETCH_H, METPREFETCH_M. */
6581         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6582         if (fidx) {
6583                 CHECK(!fidx->var_size, EINVAL);
6584
6585                 instr->type = INSTR_METPREFETCH_M;
6586                 if (idx[0] == 'h')
6587                         instr->type = INSTR_METPREFETCH_H;
6588
6589                 instr->meter.metarray_id = m->id;
6590                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6591                 instr->meter.idx.n_bits = fidx->n_bits;
6592                 instr->meter.idx.offset = fidx->offset / 8;
6593                 return 0;
6594         }
6595
6596         /* METPREFETCH_I. */
6597         idx_val = strtoul(idx, &idx, 0);
6598         CHECK(!idx[0], EINVAL);
6599
6600         instr->type = INSTR_METPREFETCH_I;
6601         instr->meter.metarray_id = m->id;
6602         instr->meter.idx_val = idx_val;
6603         return 0;
6604 }
6605
6606 static int
6607 instr_meter_translate(struct rte_swx_pipeline *p,
6608                       struct action *action,
6609                       char **tokens,
6610                       int n_tokens,
6611                       struct instruction *instr,
6612                       struct instruction_data *data __rte_unused)
6613 {
6614         char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
6615         char *color_in = tokens[4], *color_out = tokens[5];
6616         struct metarray *m;
6617         struct field *fidx, *flength, *fcin, *fcout;
6618         uint32_t idx_struct_id, length_struct_id;
6619         uint32_t color_in_struct_id, color_out_struct_id;
6620
6621         CHECK(n_tokens == 6, EINVAL);
6622
6623         m = metarray_find(p, metarray);
6624         CHECK(m, EINVAL);
6625
6626         fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6627
6628         flength = struct_field_parse(p, action, length, &length_struct_id);
6629         CHECK(flength, EINVAL);
6630         CHECK(!flength->var_size, EINVAL);
6631
6632         fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
6633
6634         fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
6635         CHECK(fcout, EINVAL);
6636         CHECK(!fcout->var_size, EINVAL);
6637
6638         /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
6639         if (fidx && fcin) {
6640                 CHECK(!fidx->var_size, EINVAL);
6641                 CHECK(!fcin->var_size, EINVAL);
6642
6643                 instr->type = INSTR_METER_MMM;
6644                 if (idx[0] == 'h' && length[0] == 'h')
6645                         instr->type = INSTR_METER_HHM;
6646                 if (idx[0] == 'h' && length[0] != 'h')
6647                         instr->type = INSTR_METER_HMM;
6648                 if (idx[0] != 'h' && length[0] == 'h')
6649                         instr->type = INSTR_METER_MHM;
6650
6651                 instr->meter.metarray_id = m->id;
6652
6653                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6654                 instr->meter.idx.n_bits = fidx->n_bits;
6655                 instr->meter.idx.offset = fidx->offset / 8;
6656
6657                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6658                 instr->meter.length.n_bits = flength->n_bits;
6659                 instr->meter.length.offset = flength->offset / 8;
6660
6661                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6662                 instr->meter.color_in.n_bits = fcin->n_bits;
6663                 instr->meter.color_in.offset = fcin->offset / 8;
6664
6665                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6666                 instr->meter.color_out.n_bits = fcout->n_bits;
6667                 instr->meter.color_out.offset = fcout->offset / 8;
6668
6669                 return 0;
6670         }
6671
6672         /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
6673         if (fidx && !fcin) {
6674                 uint32_t color_in_val;
6675
6676                 CHECK(!fidx->var_size, EINVAL);
6677
6678                 color_in_val = strtoul(color_in, &color_in, 0);
6679                 CHECK(!color_in[0], EINVAL);
6680
6681                 instr->type = INSTR_METER_MMI;
6682                 if (idx[0] == 'h' && length[0] == 'h')
6683                         instr->type = INSTR_METER_HHI;
6684                 if (idx[0] == 'h' && length[0] != 'h')
6685                         instr->type = INSTR_METER_HMI;
6686                 if (idx[0] != 'h' && length[0] == 'h')
6687                         instr->type = INSTR_METER_MHI;
6688
6689                 instr->meter.metarray_id = m->id;
6690
6691                 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6692                 instr->meter.idx.n_bits = fidx->n_bits;
6693                 instr->meter.idx.offset = fidx->offset / 8;
6694
6695                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6696                 instr->meter.length.n_bits = flength->n_bits;
6697                 instr->meter.length.offset = flength->offset / 8;
6698
6699                 instr->meter.color_in_val = color_in_val;
6700
6701                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6702                 instr->meter.color_out.n_bits = fcout->n_bits;
6703                 instr->meter.color_out.offset = fcout->offset / 8;
6704
6705                 return 0;
6706         }
6707
6708         /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
6709         if (!fidx && fcin) {
6710                 uint32_t idx_val;
6711
6712                 idx_val = strtoul(idx, &idx, 0);
6713                 CHECK(!idx[0], EINVAL);
6714
6715                 CHECK(!fcin->var_size, EINVAL);
6716
6717                 instr->type = INSTR_METER_IMM;
6718                 if (length[0] == 'h')
6719                         instr->type = INSTR_METER_IHM;
6720
6721                 instr->meter.metarray_id = m->id;
6722
6723                 instr->meter.idx_val = idx_val;
6724
6725                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6726                 instr->meter.length.n_bits = flength->n_bits;
6727                 instr->meter.length.offset = flength->offset / 8;
6728
6729                 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6730                 instr->meter.color_in.n_bits = fcin->n_bits;
6731                 instr->meter.color_in.offset = fcin->offset / 8;
6732
6733                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6734                 instr->meter.color_out.n_bits = fcout->n_bits;
6735                 instr->meter.color_out.offset = fcout->offset / 8;
6736
6737                 return 0;
6738         }
6739
6740         /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
6741         if (!fidx && !fcin) {
6742                 uint32_t idx_val, color_in_val;
6743
6744                 idx_val = strtoul(idx, &idx, 0);
6745                 CHECK(!idx[0], EINVAL);
6746
6747                 color_in_val = strtoul(color_in, &color_in, 0);
6748                 CHECK(!color_in[0], EINVAL);
6749
6750                 instr->type = INSTR_METER_IMI;
6751                 if (length[0] == 'h')
6752                         instr->type = INSTR_METER_IHI;
6753
6754                 instr->meter.metarray_id = m->id;
6755
6756                 instr->meter.idx_val = idx_val;
6757
6758                 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6759                 instr->meter.length.n_bits = flength->n_bits;
6760                 instr->meter.length.offset = flength->offset / 8;
6761
6762                 instr->meter.color_in_val = color_in_val;
6763
6764                 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6765                 instr->meter.color_out.n_bits = fcout->n_bits;
6766                 instr->meter.color_out.offset = fcout->offset / 8;
6767
6768                 return 0;
6769         }
6770
6771         CHECK(0, EINVAL);
6772 }
6773
6774 static inline struct meter *
6775 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6776 {
6777         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6778
6779         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6780         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6781         uint64_t idx64 = *idx64_ptr;
6782         uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
6783         uint64_t idx = idx64 & idx64_mask & r->size_mask;
6784
6785         return &r->metarray[idx];
6786 }
6787
6788 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6789
6790 static inline struct meter *
6791 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6792 {
6793         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6794
6795         uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6796         uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6797         uint64_t idx64 = *idx64_ptr;
6798         uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
6799
6800         return &r->metarray[idx];
6801 }
6802
6803 #else
6804
6805 #define instr_meter_idx_nbo instr_meter_idx_hbo
6806
6807 #endif
6808
6809 static inline struct meter *
6810 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
6811 {
6812         struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6813
6814         uint64_t idx =  ip->meter.idx_val & r->size_mask;
6815
6816         return &r->metarray[idx];
6817 }
6818
6819 static inline uint32_t
6820 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
6821 {
6822         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6823         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6824         uint64_t src64 = *src64_ptr;
6825         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
6826         uint64_t src = src64 & src64_mask;
6827
6828         return (uint32_t)src;
6829 }
6830
6831 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6832
6833 static inline uint32_t
6834 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
6835 {
6836         uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6837         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6838         uint64_t src64 = *src64_ptr;
6839         uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
6840
6841         return (uint32_t)src;
6842 }
6843
6844 #else
6845
6846 #define instr_meter_length_nbo instr_meter_length_hbo
6847
6848 #endif
6849
6850 static inline enum rte_color
6851 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
6852 {
6853         uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
6854         uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
6855         uint64_t src64 = *src64_ptr;
6856         uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
6857         uint64_t src = src64 & src64_mask;
6858
6859         return (enum rte_color)src;
6860 }
6861
6862 static inline void
6863 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
6864 {
6865         uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
6866         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
6867         uint64_t dst64 = *dst64_ptr;
6868         uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
6869
6870         uint64_t src = (uint64_t)color_out;
6871
6872         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6873 }
6874
6875 static inline void
6876 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
6877 {
6878         struct thread *t = &p->threads[p->thread_id];
6879         struct instruction *ip = t->ip;
6880         struct meter *m;
6881
6882         TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
6883
6884         /* Structs. */
6885         m = instr_meter_idx_nbo(p, t, ip);
6886         rte_prefetch0(m);
6887
6888         /* Thread. */
6889         thread_ip_inc(p);
6890 }
6891
6892 static inline void
6893 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
6894 {
6895         struct thread *t = &p->threads[p->thread_id];
6896         struct instruction *ip = t->ip;
6897         struct meter *m;
6898
6899         TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
6900
6901         /* Structs. */
6902         m = instr_meter_idx_hbo(p, t, ip);
6903         rte_prefetch0(m);
6904
6905         /* Thread. */
6906         thread_ip_inc(p);
6907 }
6908
6909 static inline void
6910 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
6911 {
6912         struct thread *t = &p->threads[p->thread_id];
6913         struct instruction *ip = t->ip;
6914         struct meter *m;
6915
6916         TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
6917
6918         /* Structs. */
6919         m = instr_meter_idx_imm(p, ip);
6920         rte_prefetch0(m);
6921
6922         /* Thread. */
6923         thread_ip_inc(p);
6924 }
6925
6926 static inline void
6927 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
6928 {
6929         struct thread *t = &p->threads[p->thread_id];
6930         struct instruction *ip = t->ip;
6931         struct meter *m;
6932         uint64_t time, n_pkts, n_bytes;
6933         uint32_t length;
6934         enum rte_color color_in, color_out;
6935
6936         TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
6937
6938         /* Structs. */
6939         m = instr_meter_idx_nbo(p, t, ip);
6940         rte_prefetch0(m->n_pkts);
6941         time = rte_get_tsc_cycles();
6942         length = instr_meter_length_nbo(t, ip);
6943         color_in = instr_meter_color_in_hbo(t, ip);
6944
6945         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6946                 &m->profile->profile,
6947                 time,
6948                 length,
6949                 color_in);
6950
6951         color_out &= m->color_mask;
6952
6953         n_pkts = m->n_pkts[color_out];
6954         n_bytes = m->n_bytes[color_out];
6955
6956         instr_meter_color_out_hbo_set(t, ip, color_out);
6957
6958         m->n_pkts[color_out] = n_pkts + 1;
6959         m->n_bytes[color_out] = n_bytes + length;
6960
6961         /* Thread. */
6962         thread_ip_inc(p);
6963 }
6964
6965 static inline void
6966 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
6967 {
6968         struct thread *t = &p->threads[p->thread_id];
6969         struct instruction *ip = t->ip;
6970         struct meter *m;
6971         uint64_t time, n_pkts, n_bytes;
6972         uint32_t length;
6973         enum rte_color color_in, color_out;
6974
6975         TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
6976
6977         /* Structs. */
6978         m = instr_meter_idx_nbo(p, t, ip);
6979         rte_prefetch0(m->n_pkts);
6980         time = rte_get_tsc_cycles();
6981         length = instr_meter_length_nbo(t, ip);
6982         color_in = (enum rte_color)ip->meter.color_in_val;
6983
6984         color_out = rte_meter_trtcm_color_aware_check(&m->m,
6985                 &m->profile->profile,
6986                 time,
6987                 length,
6988                 color_in);
6989
6990         color_out &= m->color_mask;
6991
6992         n_pkts = m->n_pkts[color_out];
6993         n_bytes = m->n_bytes[color_out];
6994
6995         instr_meter_color_out_hbo_set(t, ip, color_out);
6996
6997         m->n_pkts[color_out] = n_pkts + 1;
6998         m->n_bytes[color_out] = n_bytes + length;
6999
7000         /* Thread. */
7001         thread_ip_inc(p);
7002 }
7003
7004 static inline void
7005 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
7006 {
7007         struct thread *t = &p->threads[p->thread_id];
7008         struct instruction *ip = t->ip;
7009         struct meter *m;
7010         uint64_t time, n_pkts, n_bytes;
7011         uint32_t length;
7012         enum rte_color color_in, color_out;
7013
7014         TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
7015
7016         /* Structs. */
7017         m = instr_meter_idx_nbo(p, t, ip);
7018         rte_prefetch0(m->n_pkts);
7019         time = rte_get_tsc_cycles();
7020         length = instr_meter_length_hbo(t, ip);
7021         color_in = instr_meter_color_in_hbo(t, ip);
7022
7023         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7024                 &m->profile->profile,
7025                 time,
7026                 length,
7027                 color_in);
7028
7029         color_out &= m->color_mask;
7030
7031         n_pkts = m->n_pkts[color_out];
7032         n_bytes = m->n_bytes[color_out];
7033
7034         instr_meter_color_out_hbo_set(t, ip, color_out);
7035
7036         m->n_pkts[color_out] = n_pkts + 1;
7037         m->n_bytes[color_out] = n_bytes + length;
7038
7039         /* Thread. */
7040         thread_ip_inc(p);
7041 }
7042 static inline void
7043 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
7044 {
7045         struct thread *t = &p->threads[p->thread_id];
7046         struct instruction *ip = t->ip;
7047         struct meter *m;
7048         uint64_t time, n_pkts, n_bytes;
7049         uint32_t length;
7050         enum rte_color color_in, color_out;
7051
7052         TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
7053
7054         /* Structs. */
7055         m = instr_meter_idx_nbo(p, t, ip);
7056         rte_prefetch0(m->n_pkts);
7057         time = rte_get_tsc_cycles();
7058         length = instr_meter_length_hbo(t, ip);
7059         color_in = (enum rte_color)ip->meter.color_in_val;
7060
7061         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7062                 &m->profile->profile,
7063                 time,
7064                 length,
7065                 color_in);
7066
7067         color_out &= m->color_mask;
7068
7069         n_pkts = m->n_pkts[color_out];
7070         n_bytes = m->n_bytes[color_out];
7071
7072         instr_meter_color_out_hbo_set(t, ip, color_out);
7073
7074         m->n_pkts[color_out] = n_pkts + 1;
7075         m->n_bytes[color_out] = n_bytes + length;
7076
7077         /* Thread. */
7078         thread_ip_inc(p);
7079 }
7080
7081 static inline void
7082 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
7083 {
7084         struct thread *t = &p->threads[p->thread_id];
7085         struct instruction *ip = t->ip;
7086         struct meter *m;
7087         uint64_t time, n_pkts, n_bytes;
7088         uint32_t length;
7089         enum rte_color color_in, color_out;
7090
7091         TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
7092
7093         /* Structs. */
7094         m = instr_meter_idx_hbo(p, t, ip);
7095         rte_prefetch0(m->n_pkts);
7096         time = rte_get_tsc_cycles();
7097         length = instr_meter_length_nbo(t, ip);
7098         color_in = instr_meter_color_in_hbo(t, ip);
7099
7100         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7101                 &m->profile->profile,
7102                 time,
7103                 length,
7104                 color_in);
7105
7106         color_out &= m->color_mask;
7107
7108         n_pkts = m->n_pkts[color_out];
7109         n_bytes = m->n_bytes[color_out];
7110
7111         instr_meter_color_out_hbo_set(t, ip, color_out);
7112
7113         m->n_pkts[color_out] = n_pkts + 1;
7114         m->n_bytes[color_out] = n_bytes + length;
7115
7116         /* Thread. */
7117         thread_ip_inc(p);
7118 }
7119
7120 static inline void
7121 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
7122 {
7123         struct thread *t = &p->threads[p->thread_id];
7124         struct instruction *ip = t->ip;
7125         struct meter *m;
7126         uint64_t time, n_pkts, n_bytes;
7127         uint32_t length;
7128         enum rte_color color_in, color_out;
7129
7130         TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
7131
7132         /* Structs. */
7133         m = instr_meter_idx_hbo(p, t, ip);
7134         rte_prefetch0(m->n_pkts);
7135         time = rte_get_tsc_cycles();
7136         length = instr_meter_length_nbo(t, ip);
7137         color_in = (enum rte_color)ip->meter.color_in_val;
7138
7139         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7140                 &m->profile->profile,
7141                 time,
7142                 length,
7143                 color_in);
7144
7145         color_out &= m->color_mask;
7146
7147         n_pkts = m->n_pkts[color_out];
7148         n_bytes = m->n_bytes[color_out];
7149
7150         instr_meter_color_out_hbo_set(t, ip, color_out);
7151
7152         m->n_pkts[color_out] = n_pkts + 1;
7153         m->n_bytes[color_out] = n_bytes + length;
7154
7155         /* Thread. */
7156         thread_ip_inc(p);
7157 }
7158
7159 static inline void
7160 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
7161 {
7162         struct thread *t = &p->threads[p->thread_id];
7163         struct instruction *ip = t->ip;
7164         struct meter *m;
7165         uint64_t time, n_pkts, n_bytes;
7166         uint32_t length;
7167         enum rte_color color_in, color_out;
7168
7169         TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
7170
7171         /* Structs. */
7172         m = instr_meter_idx_hbo(p, t, ip);
7173         rte_prefetch0(m->n_pkts);
7174         time = rte_get_tsc_cycles();
7175         length = instr_meter_length_hbo(t, ip);
7176         color_in = instr_meter_color_in_hbo(t, ip);
7177
7178         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7179                 &m->profile->profile,
7180                 time,
7181                 length,
7182                 color_in);
7183
7184         color_out &= m->color_mask;
7185
7186         n_pkts = m->n_pkts[color_out];
7187         n_bytes = m->n_bytes[color_out];
7188
7189         instr_meter_color_out_hbo_set(t, ip, color_out);
7190
7191         m->n_pkts[color_out] = n_pkts + 1;
7192         m->n_bytes[color_out] = n_bytes + length;
7193
7194         /* Thread. */
7195         thread_ip_inc(p);
7196 }
7197
7198 static inline void
7199 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
7200 {
7201         struct thread *t = &p->threads[p->thread_id];
7202         struct instruction *ip = t->ip;
7203         struct meter *m;
7204         uint64_t time, n_pkts, n_bytes;
7205         uint32_t length;
7206         enum rte_color color_in, color_out;
7207
7208         TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
7209
7210         /* Structs. */
7211         m = instr_meter_idx_hbo(p, t, ip);
7212         rte_prefetch0(m->n_pkts);
7213         time = rte_get_tsc_cycles();
7214         length = instr_meter_length_hbo(t, ip);
7215         color_in = (enum rte_color)ip->meter.color_in_val;
7216
7217         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7218                 &m->profile->profile,
7219                 time,
7220                 length,
7221                 color_in);
7222
7223         color_out &= m->color_mask;
7224
7225         n_pkts = m->n_pkts[color_out];
7226         n_bytes = m->n_bytes[color_out];
7227
7228         instr_meter_color_out_hbo_set(t, ip, color_out);
7229
7230         m->n_pkts[color_out] = n_pkts + 1;
7231         m->n_bytes[color_out] = n_bytes + length;
7232
7233         /* Thread. */
7234         thread_ip_inc(p);
7235 }
7236
7237 static inline void
7238 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
7239 {
7240         struct thread *t = &p->threads[p->thread_id];
7241         struct instruction *ip = t->ip;
7242         struct meter *m;
7243         uint64_t time, n_pkts, n_bytes;
7244         uint32_t length;
7245         enum rte_color color_in, color_out;
7246
7247         TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
7248
7249         /* Structs. */
7250         m = instr_meter_idx_imm(p, ip);
7251         rte_prefetch0(m->n_pkts);
7252         time = rte_get_tsc_cycles();
7253         length = instr_meter_length_nbo(t, ip);
7254         color_in = instr_meter_color_in_hbo(t, ip);
7255
7256         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7257                 &m->profile->profile,
7258                 time,
7259                 length,
7260                 color_in);
7261
7262         color_out &= m->color_mask;
7263
7264         n_pkts = m->n_pkts[color_out];
7265         n_bytes = m->n_bytes[color_out];
7266
7267         instr_meter_color_out_hbo_set(t, ip, color_out);
7268
7269         m->n_pkts[color_out] = n_pkts + 1;
7270         m->n_bytes[color_out] = n_bytes + length;
7271
7272         /* Thread. */
7273         thread_ip_inc(p);
7274 }
7275
7276 static inline void
7277 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
7278 {
7279         struct thread *t = &p->threads[p->thread_id];
7280         struct instruction *ip = t->ip;
7281         struct meter *m;
7282         uint64_t time, n_pkts, n_bytes;
7283         uint32_t length;
7284         enum rte_color color_in, color_out;
7285
7286         TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
7287
7288         /* Structs. */
7289         m = instr_meter_idx_imm(p, ip);
7290         rte_prefetch0(m->n_pkts);
7291         time = rte_get_tsc_cycles();
7292         length = instr_meter_length_nbo(t, ip);
7293         color_in = (enum rte_color)ip->meter.color_in_val;
7294
7295         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7296                 &m->profile->profile,
7297                 time,
7298                 length,
7299                 color_in);
7300
7301         color_out &= m->color_mask;
7302
7303         n_pkts = m->n_pkts[color_out];
7304         n_bytes = m->n_bytes[color_out];
7305
7306         instr_meter_color_out_hbo_set(t, ip, color_out);
7307
7308         m->n_pkts[color_out] = n_pkts + 1;
7309         m->n_bytes[color_out] = n_bytes + length;
7310
7311         /* Thread. */
7312         thread_ip_inc(p);
7313 }
7314
7315 static inline void
7316 instr_meter_imm_exec(struct rte_swx_pipeline *p)
7317 {
7318         struct thread *t = &p->threads[p->thread_id];
7319         struct instruction *ip = t->ip;
7320         struct meter *m;
7321         uint64_t time, n_pkts, n_bytes;
7322         uint32_t length;
7323         enum rte_color color_in, color_out;
7324
7325         TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
7326
7327         /* Structs. */
7328         m = instr_meter_idx_imm(p, ip);
7329         rte_prefetch0(m->n_pkts);
7330         time = rte_get_tsc_cycles();
7331         length = instr_meter_length_hbo(t, ip);
7332         color_in = instr_meter_color_in_hbo(t, ip);
7333
7334         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7335                 &m->profile->profile,
7336                 time,
7337                 length,
7338                 color_in);
7339
7340         color_out &= m->color_mask;
7341
7342         n_pkts = m->n_pkts[color_out];
7343         n_bytes = m->n_bytes[color_out];
7344
7345         instr_meter_color_out_hbo_set(t, ip, color_out);
7346
7347         m->n_pkts[color_out] = n_pkts + 1;
7348         m->n_bytes[color_out] = n_bytes + length;
7349
7350         /* Thread. */
7351         thread_ip_inc(p);
7352 }
7353 static inline void
7354 instr_meter_imi_exec(struct rte_swx_pipeline *p)
7355 {
7356         struct thread *t = &p->threads[p->thread_id];
7357         struct instruction *ip = t->ip;
7358         struct meter *m;
7359         uint64_t time, n_pkts, n_bytes;
7360         uint32_t length;
7361         enum rte_color color_in, color_out;
7362
7363         TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
7364
7365         /* Structs. */
7366         m = instr_meter_idx_imm(p, ip);
7367         rte_prefetch0(m->n_pkts);
7368         time = rte_get_tsc_cycles();
7369         length = instr_meter_length_hbo(t, ip);
7370         color_in = (enum rte_color)ip->meter.color_in_val;
7371
7372         color_out = rte_meter_trtcm_color_aware_check(&m->m,
7373                 &m->profile->profile,
7374                 time,
7375                 length,
7376                 color_in);
7377
7378         color_out &= m->color_mask;
7379
7380         n_pkts = m->n_pkts[color_out];
7381         n_bytes = m->n_bytes[color_out];
7382
7383         instr_meter_color_out_hbo_set(t, ip, color_out);
7384
7385         m->n_pkts[color_out] = n_pkts + 1;
7386         m->n_bytes[color_out] = n_bytes + length;
7387
7388         /* Thread. */
7389         thread_ip_inc(p);
7390 }
7391
7392 /*
7393  * jmp.
7394  */
7395 static int
7396 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
7397                     struct action *action __rte_unused,
7398                     char **tokens,
7399                     int n_tokens,
7400                     struct instruction *instr,
7401                     struct instruction_data *data)
7402 {
7403         CHECK(n_tokens == 2, EINVAL);
7404
7405         strcpy(data->jmp_label, tokens[1]);
7406
7407         instr->type = INSTR_JMP;
7408         instr->jmp.ip = NULL; /* Resolved later. */
7409         return 0;
7410 }
7411
7412 static int
7413 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
7414                           struct action *action __rte_unused,
7415                           char **tokens,
7416                           int n_tokens,
7417                           struct instruction *instr,
7418                           struct instruction_data *data)
7419 {
7420         struct header *h;
7421
7422         CHECK(n_tokens == 3, EINVAL);
7423
7424         strcpy(data->jmp_label, tokens[1]);
7425
7426         h = header_parse(p, tokens[2]);
7427         CHECK(h, EINVAL);
7428
7429         instr->type = INSTR_JMP_VALID;
7430         instr->jmp.ip = NULL; /* Resolved later. */
7431         instr->jmp.header_id = h->id;
7432         return 0;
7433 }
7434
7435 static int
7436 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
7437                             struct action *action __rte_unused,
7438                             char **tokens,
7439                             int n_tokens,
7440                             struct instruction *instr,
7441                             struct instruction_data *data)
7442 {
7443         struct header *h;
7444
7445         CHECK(n_tokens == 3, EINVAL);
7446
7447         strcpy(data->jmp_label, tokens[1]);
7448
7449         h = header_parse(p, tokens[2]);
7450         CHECK(h, EINVAL);
7451
7452         instr->type = INSTR_JMP_INVALID;
7453         instr->jmp.ip = NULL; /* Resolved later. */
7454         instr->jmp.header_id = h->id;
7455         return 0;
7456 }
7457
7458 static int
7459 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
7460                         struct action *action,
7461                         char **tokens,
7462                         int n_tokens,
7463                         struct instruction *instr,
7464                         struct instruction_data *data)
7465 {
7466         CHECK(!action, EINVAL);
7467         CHECK(n_tokens == 2, EINVAL);
7468
7469         strcpy(data->jmp_label, tokens[1]);
7470
7471         instr->type = INSTR_JMP_HIT;
7472         instr->jmp.ip = NULL; /* Resolved later. */
7473         return 0;
7474 }
7475
7476 static int
7477 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
7478                          struct action *action,
7479                          char **tokens,
7480                          int n_tokens,
7481                          struct instruction *instr,
7482                          struct instruction_data *data)
7483 {
7484         CHECK(!action, EINVAL);
7485         CHECK(n_tokens == 2, EINVAL);
7486
7487         strcpy(data->jmp_label, tokens[1]);
7488
7489         instr->type = INSTR_JMP_MISS;
7490         instr->jmp.ip = NULL; /* Resolved later. */
7491         return 0;
7492 }
7493
7494 static int
7495 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
7496                                struct action *action,
7497                                char **tokens,
7498                                int n_tokens,
7499                                struct instruction *instr,
7500                                struct instruction_data *data)
7501 {
7502         struct action *a;
7503
7504         CHECK(!action, EINVAL);
7505         CHECK(n_tokens == 3, EINVAL);
7506
7507         strcpy(data->jmp_label, tokens[1]);
7508
7509         a = action_find(p, tokens[2]);
7510         CHECK(a, EINVAL);
7511
7512         instr->type = INSTR_JMP_ACTION_HIT;
7513         instr->jmp.ip = NULL; /* Resolved later. */
7514         instr->jmp.action_id = a->id;
7515         return 0;
7516 }
7517
7518 static int
7519 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
7520                                 struct action *action,
7521                                 char **tokens,
7522                                 int n_tokens,
7523                                 struct instruction *instr,
7524                                 struct instruction_data *data)
7525 {
7526         struct action *a;
7527
7528         CHECK(!action, EINVAL);
7529         CHECK(n_tokens == 3, EINVAL);
7530
7531         strcpy(data->jmp_label, tokens[1]);
7532
7533         a = action_find(p, tokens[2]);
7534         CHECK(a, EINVAL);
7535
7536         instr->type = INSTR_JMP_ACTION_MISS;
7537         instr->jmp.ip = NULL; /* Resolved later. */
7538         instr->jmp.action_id = a->id;
7539         return 0;
7540 }
7541
7542 static int
7543 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
7544                        struct action *action,
7545                        char **tokens,
7546                        int n_tokens,
7547                        struct instruction *instr,
7548                        struct instruction_data *data)
7549 {
7550         char *a = tokens[2], *b = tokens[3];
7551         struct field *fa, *fb;
7552         uint64_t b_val;
7553         uint32_t a_struct_id, b_struct_id;
7554
7555         CHECK(n_tokens == 4, EINVAL);
7556
7557         strcpy(data->jmp_label, tokens[1]);
7558
7559         fa = struct_field_parse(p, action, a, &a_struct_id);
7560         CHECK(fa, EINVAL);
7561         CHECK(!fa->var_size, EINVAL);
7562
7563         /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
7564         fb = struct_field_parse(p, action, b, &b_struct_id);
7565         if (fb) {
7566                 CHECK(!fb->var_size, EINVAL);
7567
7568                 instr->type = INSTR_JMP_EQ;
7569                 if (a[0] != 'h' && b[0] == 'h')
7570                         instr->type = INSTR_JMP_EQ_MH;
7571                 if (a[0] == 'h' && b[0] != 'h')
7572                         instr->type = INSTR_JMP_EQ_HM;
7573                 if (a[0] == 'h' && b[0] == 'h')
7574                         instr->type = INSTR_JMP_EQ_HH;
7575                 instr->jmp.ip = NULL; /* Resolved later. */
7576
7577                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7578                 instr->jmp.a.n_bits = fa->n_bits;
7579                 instr->jmp.a.offset = fa->offset / 8;
7580                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7581                 instr->jmp.b.n_bits = fb->n_bits;
7582                 instr->jmp.b.offset = fb->offset / 8;
7583                 return 0;
7584         }
7585
7586         /* JMP_EQ_I. */
7587         b_val = strtoull(b, &b, 0);
7588         CHECK(!b[0], EINVAL);
7589
7590         if (a[0] == 'h')
7591                 b_val = hton64(b_val) >> (64 - fa->n_bits);
7592
7593         instr->type = INSTR_JMP_EQ_I;
7594         instr->jmp.ip = NULL; /* Resolved later. */
7595         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7596         instr->jmp.a.n_bits = fa->n_bits;
7597         instr->jmp.a.offset = fa->offset / 8;
7598         instr->jmp.b_val = b_val;
7599         return 0;
7600 }
7601
7602 static int
7603 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
7604                         struct action *action,
7605                         char **tokens,
7606                         int n_tokens,
7607                         struct instruction *instr,
7608                         struct instruction_data *data)
7609 {
7610         char *a = tokens[2], *b = tokens[3];
7611         struct field *fa, *fb;
7612         uint64_t b_val;
7613         uint32_t a_struct_id, b_struct_id;
7614
7615         CHECK(n_tokens == 4, EINVAL);
7616
7617         strcpy(data->jmp_label, tokens[1]);
7618
7619         fa = struct_field_parse(p, action, a, &a_struct_id);
7620         CHECK(fa, EINVAL);
7621         CHECK(!fa->var_size, EINVAL);
7622
7623         /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
7624         fb = struct_field_parse(p, action, b, &b_struct_id);
7625         if (fb) {
7626                 CHECK(!fb->var_size, EINVAL);
7627
7628                 instr->type = INSTR_JMP_NEQ;
7629                 if (a[0] != 'h' && b[0] == 'h')
7630                         instr->type = INSTR_JMP_NEQ_MH;
7631                 if (a[0] == 'h' && b[0] != 'h')
7632                         instr->type = INSTR_JMP_NEQ_HM;
7633                 if (a[0] == 'h' && b[0] == 'h')
7634                         instr->type = INSTR_JMP_NEQ_HH;
7635                 instr->jmp.ip = NULL; /* Resolved later. */
7636
7637                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7638                 instr->jmp.a.n_bits = fa->n_bits;
7639                 instr->jmp.a.offset = fa->offset / 8;
7640                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7641                 instr->jmp.b.n_bits = fb->n_bits;
7642                 instr->jmp.b.offset = fb->offset / 8;
7643                 return 0;
7644         }
7645
7646         /* JMP_NEQ_I. */
7647         b_val = strtoull(b, &b, 0);
7648         CHECK(!b[0], EINVAL);
7649
7650         if (a[0] == 'h')
7651                 b_val = hton64(b_val) >> (64 - fa->n_bits);
7652
7653         instr->type = INSTR_JMP_NEQ_I;
7654         instr->jmp.ip = NULL; /* Resolved later. */
7655         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7656         instr->jmp.a.n_bits = fa->n_bits;
7657         instr->jmp.a.offset = fa->offset / 8;
7658         instr->jmp.b_val = b_val;
7659         return 0;
7660 }
7661
7662 static int
7663 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
7664                        struct action *action,
7665                        char **tokens,
7666                        int n_tokens,
7667                        struct instruction *instr,
7668                        struct instruction_data *data)
7669 {
7670         char *a = tokens[2], *b = tokens[3];
7671         struct field *fa, *fb;
7672         uint64_t b_val;
7673         uint32_t a_struct_id, b_struct_id;
7674
7675         CHECK(n_tokens == 4, EINVAL);
7676
7677         strcpy(data->jmp_label, tokens[1]);
7678
7679         fa = struct_field_parse(p, action, a, &a_struct_id);
7680         CHECK(fa, EINVAL);
7681         CHECK(!fa->var_size, EINVAL);
7682
7683         /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
7684         fb = struct_field_parse(p, action, b, &b_struct_id);
7685         if (fb) {
7686                 CHECK(!fb->var_size, EINVAL);
7687
7688                 instr->type = INSTR_JMP_LT;
7689                 if (a[0] == 'h' && b[0] != 'h')
7690                         instr->type = INSTR_JMP_LT_HM;
7691                 if (a[0] != 'h' && b[0] == 'h')
7692                         instr->type = INSTR_JMP_LT_MH;
7693                 if (a[0] == 'h' && b[0] == 'h')
7694                         instr->type = INSTR_JMP_LT_HH;
7695                 instr->jmp.ip = NULL; /* Resolved later. */
7696
7697                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7698                 instr->jmp.a.n_bits = fa->n_bits;
7699                 instr->jmp.a.offset = fa->offset / 8;
7700                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7701                 instr->jmp.b.n_bits = fb->n_bits;
7702                 instr->jmp.b.offset = fb->offset / 8;
7703                 return 0;
7704         }
7705
7706         /* JMP_LT_MI, JMP_LT_HI. */
7707         b_val = strtoull(b, &b, 0);
7708         CHECK(!b[0], EINVAL);
7709
7710         instr->type = INSTR_JMP_LT_MI;
7711         if (a[0] == 'h')
7712                 instr->type = INSTR_JMP_LT_HI;
7713         instr->jmp.ip = NULL; /* Resolved later. */
7714
7715         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7716         instr->jmp.a.n_bits = fa->n_bits;
7717         instr->jmp.a.offset = fa->offset / 8;
7718         instr->jmp.b_val = b_val;
7719         return 0;
7720 }
7721
7722 static int
7723 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
7724                        struct action *action,
7725                        char **tokens,
7726                        int n_tokens,
7727                        struct instruction *instr,
7728                        struct instruction_data *data)
7729 {
7730         char *a = tokens[2], *b = tokens[3];
7731         struct field *fa, *fb;
7732         uint64_t b_val;
7733         uint32_t a_struct_id, b_struct_id;
7734
7735         CHECK(n_tokens == 4, EINVAL);
7736
7737         strcpy(data->jmp_label, tokens[1]);
7738
7739         fa = struct_field_parse(p, action, a, &a_struct_id);
7740         CHECK(fa, EINVAL);
7741         CHECK(!fa->var_size, EINVAL);
7742
7743         /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
7744         fb = struct_field_parse(p, action, b, &b_struct_id);
7745         if (fb) {
7746                 CHECK(!fb->var_size, EINVAL);
7747
7748                 instr->type = INSTR_JMP_GT;
7749                 if (a[0] == 'h' && b[0] != 'h')
7750                         instr->type = INSTR_JMP_GT_HM;
7751                 if (a[0] != 'h' && b[0] == 'h')
7752                         instr->type = INSTR_JMP_GT_MH;
7753                 if (a[0] == 'h' && b[0] == 'h')
7754                         instr->type = INSTR_JMP_GT_HH;
7755                 instr->jmp.ip = NULL; /* Resolved later. */
7756
7757                 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7758                 instr->jmp.a.n_bits = fa->n_bits;
7759                 instr->jmp.a.offset = fa->offset / 8;
7760                 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7761                 instr->jmp.b.n_bits = fb->n_bits;
7762                 instr->jmp.b.offset = fb->offset / 8;
7763                 return 0;
7764         }
7765
7766         /* JMP_GT_MI, JMP_GT_HI. */
7767         b_val = strtoull(b, &b, 0);
7768         CHECK(!b[0], EINVAL);
7769
7770         instr->type = INSTR_JMP_GT_MI;
7771         if (a[0] == 'h')
7772                 instr->type = INSTR_JMP_GT_HI;
7773         instr->jmp.ip = NULL; /* Resolved later. */
7774
7775         instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7776         instr->jmp.a.n_bits = fa->n_bits;
7777         instr->jmp.a.offset = fa->offset / 8;
7778         instr->jmp.b_val = b_val;
7779         return 0;
7780 }
7781
7782 static inline void
7783 instr_jmp_exec(struct rte_swx_pipeline *p)
7784 {
7785         struct thread *t = &p->threads[p->thread_id];
7786         struct instruction *ip = t->ip;
7787
7788         TRACE("[Thread %2u] jmp\n", p->thread_id);
7789
7790         thread_ip_set(t, ip->jmp.ip);
7791 }
7792
7793 static inline void
7794 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
7795 {
7796         struct thread *t = &p->threads[p->thread_id];
7797         struct instruction *ip = t->ip;
7798         uint32_t header_id = ip->jmp.header_id;
7799
7800         TRACE("[Thread %2u] jmpv\n", p->thread_id);
7801
7802         t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
7803 }
7804
7805 static inline void
7806 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
7807 {
7808         struct thread *t = &p->threads[p->thread_id];
7809         struct instruction *ip = t->ip;
7810         uint32_t header_id = ip->jmp.header_id;
7811
7812         TRACE("[Thread %2u] jmpnv\n", p->thread_id);
7813
7814         t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
7815 }
7816
7817 static inline void
7818 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
7819 {
7820         struct thread *t = &p->threads[p->thread_id];
7821         struct instruction *ip = t->ip;
7822         struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
7823
7824         TRACE("[Thread %2u] jmph\n", p->thread_id);
7825
7826         t->ip = ip_next[t->hit];
7827 }
7828
7829 static inline void
7830 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
7831 {
7832         struct thread *t = &p->threads[p->thread_id];
7833         struct instruction *ip = t->ip;
7834         struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
7835
7836         TRACE("[Thread %2u] jmpnh\n", p->thread_id);
7837
7838         t->ip = ip_next[t->hit];
7839 }
7840
7841 static inline void
7842 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
7843 {
7844         struct thread *t = &p->threads[p->thread_id];
7845         struct instruction *ip = t->ip;
7846
7847         TRACE("[Thread %2u] jmpa\n", p->thread_id);
7848
7849         t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
7850 }
7851
7852 static inline void
7853 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
7854 {
7855         struct thread *t = &p->threads[p->thread_id];
7856         struct instruction *ip = t->ip;
7857
7858         TRACE("[Thread %2u] jmpna\n", p->thread_id);
7859
7860         t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
7861 }
7862
7863 static inline void
7864 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
7865 {
7866         struct thread *t = &p->threads[p->thread_id];
7867         struct instruction *ip = t->ip;
7868
7869         TRACE("[Thread %2u] jmpeq\n", p->thread_id);
7870
7871         JMP_CMP(t, ip, ==);
7872 }
7873
7874 static inline void
7875 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
7876 {
7877         struct thread *t = &p->threads[p->thread_id];
7878         struct instruction *ip = t->ip;
7879
7880         TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
7881
7882         JMP_CMP_MH(t, ip, ==);
7883 }
7884
7885 static inline void
7886 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
7887 {
7888         struct thread *t = &p->threads[p->thread_id];
7889         struct instruction *ip = t->ip;
7890
7891         TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
7892
7893         JMP_CMP_HM(t, ip, ==);
7894 }
7895
7896 static inline void
7897 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
7898 {
7899         struct thread *t = &p->threads[p->thread_id];
7900         struct instruction *ip = t->ip;
7901
7902         TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
7903
7904         JMP_CMP_HH_FAST(t, ip, ==);
7905 }
7906
7907 static inline void
7908 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
7909 {
7910         struct thread *t = &p->threads[p->thread_id];
7911         struct instruction *ip = t->ip;
7912
7913         TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
7914
7915         JMP_CMP_I(t, ip, ==);
7916 }
7917
7918 static inline void
7919 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
7920 {
7921         struct thread *t = &p->threads[p->thread_id];
7922         struct instruction *ip = t->ip;
7923
7924         TRACE("[Thread %2u] jmpneq\n", p->thread_id);
7925
7926         JMP_CMP(t, ip, !=);
7927 }
7928
7929 static inline void
7930 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
7931 {
7932         struct thread *t = &p->threads[p->thread_id];
7933         struct instruction *ip = t->ip;
7934
7935         TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
7936
7937         JMP_CMP_MH(t, ip, !=);
7938 }
7939
7940 static inline void
7941 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
7942 {
7943         struct thread *t = &p->threads[p->thread_id];
7944         struct instruction *ip = t->ip;
7945
7946         TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
7947
7948         JMP_CMP_HM(t, ip, !=);
7949 }
7950
7951 static inline void
7952 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
7953 {
7954         struct thread *t = &p->threads[p->thread_id];
7955         struct instruction *ip = t->ip;
7956
7957         TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
7958
7959         JMP_CMP_HH_FAST(t, ip, !=);
7960 }
7961
7962 static inline void
7963 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
7964 {
7965         struct thread *t = &p->threads[p->thread_id];
7966         struct instruction *ip = t->ip;
7967
7968         TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
7969
7970         JMP_CMP_I(t, ip, !=);
7971 }
7972
7973 static inline void
7974 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
7975 {
7976         struct thread *t = &p->threads[p->thread_id];
7977         struct instruction *ip = t->ip;
7978
7979         TRACE("[Thread %2u] jmplt\n", p->thread_id);
7980
7981         JMP_CMP(t, ip, <);
7982 }
7983
7984 static inline void
7985 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
7986 {
7987         struct thread *t = &p->threads[p->thread_id];
7988         struct instruction *ip = t->ip;
7989
7990         TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
7991
7992         JMP_CMP_MH(t, ip, <);
7993 }
7994
7995 static inline void
7996 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
7997 {
7998         struct thread *t = &p->threads[p->thread_id];
7999         struct instruction *ip = t->ip;
8000
8001         TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
8002
8003         JMP_CMP_HM(t, ip, <);
8004 }
8005
8006 static inline void
8007 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
8008 {
8009         struct thread *t = &p->threads[p->thread_id];
8010         struct instruction *ip = t->ip;
8011
8012         TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
8013
8014         JMP_CMP_HH(t, ip, <);
8015 }
8016
8017 static inline void
8018 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
8019 {
8020         struct thread *t = &p->threads[p->thread_id];
8021         struct instruction *ip = t->ip;
8022
8023         TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
8024
8025         JMP_CMP_MI(t, ip, <);
8026 }
8027
8028 static inline void
8029 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
8030 {
8031         struct thread *t = &p->threads[p->thread_id];
8032         struct instruction *ip = t->ip;
8033
8034         TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
8035
8036         JMP_CMP_HI(t, ip, <);
8037 }
8038
8039 static inline void
8040 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
8041 {
8042         struct thread *t = &p->threads[p->thread_id];
8043         struct instruction *ip = t->ip;
8044
8045         TRACE("[Thread %2u] jmpgt\n", p->thread_id);
8046
8047         JMP_CMP(t, ip, >);
8048 }
8049
8050 static inline void
8051 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
8052 {
8053         struct thread *t = &p->threads[p->thread_id];
8054         struct instruction *ip = t->ip;
8055
8056         TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
8057
8058         JMP_CMP_MH(t, ip, >);
8059 }
8060
8061 static inline void
8062 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
8063 {
8064         struct thread *t = &p->threads[p->thread_id];
8065         struct instruction *ip = t->ip;
8066
8067         TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
8068
8069         JMP_CMP_HM(t, ip, >);
8070 }
8071
8072 static inline void
8073 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
8074 {
8075         struct thread *t = &p->threads[p->thread_id];
8076         struct instruction *ip = t->ip;
8077
8078         TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
8079
8080         JMP_CMP_HH(t, ip, >);
8081 }
8082
8083 static inline void
8084 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
8085 {
8086         struct thread *t = &p->threads[p->thread_id];
8087         struct instruction *ip = t->ip;
8088
8089         TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
8090
8091         JMP_CMP_MI(t, ip, >);
8092 }
8093
8094 static inline void
8095 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
8096 {
8097         struct thread *t = &p->threads[p->thread_id];
8098         struct instruction *ip = t->ip;
8099
8100         TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
8101
8102         JMP_CMP_HI(t, ip, >);
8103 }
8104
8105 /*
8106  * return.
8107  */
8108 static int
8109 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
8110                        struct action *action,
8111                        char **tokens __rte_unused,
8112                        int n_tokens,
8113                        struct instruction *instr,
8114                        struct instruction_data *data __rte_unused)
8115 {
8116         CHECK(action, EINVAL);
8117         CHECK(n_tokens == 1, EINVAL);
8118
8119         instr->type = INSTR_RETURN;
8120         return 0;
8121 }
8122
8123 static inline void
8124 instr_return_exec(struct rte_swx_pipeline *p)
8125 {
8126         struct thread *t = &p->threads[p->thread_id];
8127
8128         TRACE("[Thread %2u] return\n", p->thread_id);
8129
8130         t->ip = t->ret;
8131 }
8132
8133 static int
8134 instr_translate(struct rte_swx_pipeline *p,
8135                 struct action *action,
8136                 char *string,
8137                 struct instruction *instr,
8138                 struct instruction_data *data)
8139 {
8140         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
8141         int n_tokens = 0, tpos = 0;
8142
8143         /* Parse the instruction string into tokens. */
8144         for ( ; ; ) {
8145                 char *token;
8146
8147                 token = strtok_r(string, " \t\v", &string);
8148                 if (!token)
8149                         break;
8150
8151                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
8152                 CHECK_NAME(token, EINVAL);
8153
8154                 tokens[n_tokens] = token;
8155                 n_tokens++;
8156         }
8157
8158         CHECK(n_tokens, EINVAL);
8159
8160         /* Handle the optional instruction label. */
8161         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
8162                 strcpy(data->label, tokens[0]);
8163
8164                 tpos += 2;
8165                 CHECK(n_tokens - tpos, EINVAL);
8166         }
8167
8168         /* Identify the instruction type. */
8169         if (!strcmp(tokens[tpos], "rx"))
8170                 return instr_rx_translate(p,
8171                                           action,
8172                                           &tokens[tpos],
8173                                           n_tokens - tpos,
8174                                           instr,
8175                                           data);
8176
8177         if (!strcmp(tokens[tpos], "tx"))
8178                 return instr_tx_translate(p,
8179                                           action,
8180                                           &tokens[tpos],
8181                                           n_tokens - tpos,
8182                                           instr,
8183                                           data);
8184
8185         if (!strcmp(tokens[tpos], "drop"))
8186                 return instr_drop_translate(p,
8187                                             action,
8188                                             &tokens[tpos],
8189                                             n_tokens - tpos,
8190                                             instr,
8191                                             data);
8192
8193         if (!strcmp(tokens[tpos], "extract"))
8194                 return instr_hdr_extract_translate(p,
8195                                                    action,
8196                                                    &tokens[tpos],
8197                                                    n_tokens - tpos,
8198                                                    instr,
8199                                                    data);
8200
8201         if (!strcmp(tokens[tpos], "lookahead"))
8202                 return instr_hdr_lookahead_translate(p,
8203                                                      action,
8204                                                      &tokens[tpos],
8205                                                      n_tokens - tpos,
8206                                                      instr,
8207                                                      data);
8208
8209         if (!strcmp(tokens[tpos], "emit"))
8210                 return instr_hdr_emit_translate(p,
8211                                                 action,
8212                                                 &tokens[tpos],
8213                                                 n_tokens - tpos,
8214                                                 instr,
8215                                                 data);
8216
8217         if (!strcmp(tokens[tpos], "validate"))
8218                 return instr_hdr_validate_translate(p,
8219                                                     action,
8220                                                     &tokens[tpos],
8221                                                     n_tokens - tpos,
8222                                                     instr,
8223                                                     data);
8224
8225         if (!strcmp(tokens[tpos], "invalidate"))
8226                 return instr_hdr_invalidate_translate(p,
8227                                                       action,
8228                                                       &tokens[tpos],
8229                                                       n_tokens - tpos,
8230                                                       instr,
8231                                                       data);
8232
8233         if (!strcmp(tokens[tpos], "mov"))
8234                 return instr_mov_translate(p,
8235                                            action,
8236                                            &tokens[tpos],
8237                                            n_tokens - tpos,
8238                                            instr,
8239                                            data);
8240
8241         if (!strcmp(tokens[tpos], "add"))
8242                 return instr_alu_add_translate(p,
8243                                                action,
8244                                                &tokens[tpos],
8245                                                n_tokens - tpos,
8246                                                instr,
8247                                                data);
8248
8249         if (!strcmp(tokens[tpos], "sub"))
8250                 return instr_alu_sub_translate(p,
8251                                                action,
8252                                                &tokens[tpos],
8253                                                n_tokens - tpos,
8254                                                instr,
8255                                                data);
8256
8257         if (!strcmp(tokens[tpos], "ckadd"))
8258                 return instr_alu_ckadd_translate(p,
8259                                                  action,
8260                                                  &tokens[tpos],
8261                                                  n_tokens - tpos,
8262                                                  instr,
8263                                                  data);
8264
8265         if (!strcmp(tokens[tpos], "cksub"))
8266                 return instr_alu_cksub_translate(p,
8267                                                  action,
8268                                                  &tokens[tpos],
8269                                                  n_tokens - tpos,
8270                                                  instr,
8271                                                  data);
8272
8273         if (!strcmp(tokens[tpos], "and"))
8274                 return instr_alu_and_translate(p,
8275                                                action,
8276                                                &tokens[tpos],
8277                                                n_tokens - tpos,
8278                                                instr,
8279                                                data);
8280
8281         if (!strcmp(tokens[tpos], "or"))
8282                 return instr_alu_or_translate(p,
8283                                               action,
8284                                               &tokens[tpos],
8285                                               n_tokens - tpos,
8286                                               instr,
8287                                               data);
8288
8289         if (!strcmp(tokens[tpos], "xor"))
8290                 return instr_alu_xor_translate(p,
8291                                                action,
8292                                                &tokens[tpos],
8293                                                n_tokens - tpos,
8294                                                instr,
8295                                                data);
8296
8297         if (!strcmp(tokens[tpos], "shl"))
8298                 return instr_alu_shl_translate(p,
8299                                                action,
8300                                                &tokens[tpos],
8301                                                n_tokens - tpos,
8302                                                instr,
8303                                                data);
8304
8305         if (!strcmp(tokens[tpos], "shr"))
8306                 return instr_alu_shr_translate(p,
8307                                                action,
8308                                                &tokens[tpos],
8309                                                n_tokens - tpos,
8310                                                instr,
8311                                                data);
8312
8313         if (!strcmp(tokens[tpos], "regprefetch"))
8314                 return instr_regprefetch_translate(p,
8315                                                    action,
8316                                                    &tokens[tpos],
8317                                                    n_tokens - tpos,
8318                                                    instr,
8319                                                    data);
8320
8321         if (!strcmp(tokens[tpos], "regrd"))
8322                 return instr_regrd_translate(p,
8323                                              action,
8324                                              &tokens[tpos],
8325                                              n_tokens - tpos,
8326                                              instr,
8327                                              data);
8328
8329         if (!strcmp(tokens[tpos], "regwr"))
8330                 return instr_regwr_translate(p,
8331                                              action,
8332                                              &tokens[tpos],
8333                                              n_tokens - tpos,
8334                                              instr,
8335                                              data);
8336
8337         if (!strcmp(tokens[tpos], "regadd"))
8338                 return instr_regadd_translate(p,
8339                                               action,
8340                                               &tokens[tpos],
8341                                               n_tokens - tpos,
8342                                               instr,
8343                                               data);
8344
8345         if (!strcmp(tokens[tpos], "metprefetch"))
8346                 return instr_metprefetch_translate(p,
8347                                                    action,
8348                                                    &tokens[tpos],
8349                                                    n_tokens - tpos,
8350                                                    instr,
8351                                                    data);
8352
8353         if (!strcmp(tokens[tpos], "meter"))
8354                 return instr_meter_translate(p,
8355                                              action,
8356                                              &tokens[tpos],
8357                                              n_tokens - tpos,
8358                                              instr,
8359                                              data);
8360
8361         if (!strcmp(tokens[tpos], "table"))
8362                 return instr_table_translate(p,
8363                                              action,
8364                                              &tokens[tpos],
8365                                              n_tokens - tpos,
8366                                              instr,
8367                                              data);
8368
8369         if (!strcmp(tokens[tpos], "learn"))
8370                 return instr_learn_translate(p,
8371                                              action,
8372                                              &tokens[tpos],
8373                                              n_tokens - tpos,
8374                                              instr,
8375                                              data);
8376
8377         if (!strcmp(tokens[tpos], "forget"))
8378                 return instr_forget_translate(p,
8379                                               action,
8380                                               &tokens[tpos],
8381                                               n_tokens - tpos,
8382                                               instr,
8383                                               data);
8384
8385         if (!strcmp(tokens[tpos], "extern"))
8386                 return instr_extern_translate(p,
8387                                               action,
8388                                               &tokens[tpos],
8389                                               n_tokens - tpos,
8390                                               instr,
8391                                               data);
8392
8393         if (!strcmp(tokens[tpos], "jmp"))
8394                 return instr_jmp_translate(p,
8395                                            action,
8396                                            &tokens[tpos],
8397                                            n_tokens - tpos,
8398                                            instr,
8399                                            data);
8400
8401         if (!strcmp(tokens[tpos], "jmpv"))
8402                 return instr_jmp_valid_translate(p,
8403                                                  action,
8404                                                  &tokens[tpos],
8405                                                  n_tokens - tpos,
8406                                                  instr,
8407                                                  data);
8408
8409         if (!strcmp(tokens[tpos], "jmpnv"))
8410                 return instr_jmp_invalid_translate(p,
8411                                                    action,
8412                                                    &tokens[tpos],
8413                                                    n_tokens - tpos,
8414                                                    instr,
8415                                                    data);
8416
8417         if (!strcmp(tokens[tpos], "jmph"))
8418                 return instr_jmp_hit_translate(p,
8419                                                action,
8420                                                &tokens[tpos],
8421                                                n_tokens - tpos,
8422                                                instr,
8423                                                data);
8424
8425         if (!strcmp(tokens[tpos], "jmpnh"))
8426                 return instr_jmp_miss_translate(p,
8427                                                 action,
8428                                                 &tokens[tpos],
8429                                                 n_tokens - tpos,
8430                                                 instr,
8431                                                 data);
8432
8433         if (!strcmp(tokens[tpos], "jmpa"))
8434                 return instr_jmp_action_hit_translate(p,
8435                                                       action,
8436                                                       &tokens[tpos],
8437                                                       n_tokens - tpos,
8438                                                       instr,
8439                                                       data);
8440
8441         if (!strcmp(tokens[tpos], "jmpna"))
8442                 return instr_jmp_action_miss_translate(p,
8443                                                        action,
8444                                                        &tokens[tpos],
8445                                                        n_tokens - tpos,
8446                                                        instr,
8447                                                        data);
8448
8449         if (!strcmp(tokens[tpos], "jmpeq"))
8450                 return instr_jmp_eq_translate(p,
8451                                               action,
8452                                               &tokens[tpos],
8453                                               n_tokens - tpos,
8454                                               instr,
8455                                               data);
8456
8457         if (!strcmp(tokens[tpos], "jmpneq"))
8458                 return instr_jmp_neq_translate(p,
8459                                                action,
8460                                                &tokens[tpos],
8461                                                n_tokens - tpos,
8462                                                instr,
8463                                                data);
8464
8465         if (!strcmp(tokens[tpos], "jmplt"))
8466                 return instr_jmp_lt_translate(p,
8467                                               action,
8468                                               &tokens[tpos],
8469                                               n_tokens - tpos,
8470                                               instr,
8471                                               data);
8472
8473         if (!strcmp(tokens[tpos], "jmpgt"))
8474                 return instr_jmp_gt_translate(p,
8475                                               action,
8476                                               &tokens[tpos],
8477                                               n_tokens - tpos,
8478                                               instr,
8479                                               data);
8480
8481         if (!strcmp(tokens[tpos], "return"))
8482                 return instr_return_translate(p,
8483                                               action,
8484                                               &tokens[tpos],
8485                                               n_tokens - tpos,
8486                                               instr,
8487                                               data);
8488
8489         CHECK(0, EINVAL);
8490 }
8491
8492 static struct instruction_data *
8493 label_find(struct instruction_data *data, uint32_t n, const char *label)
8494 {
8495         uint32_t i;
8496
8497         for (i = 0; i < n; i++)
8498                 if (!strcmp(label, data[i].label))
8499                         return &data[i];
8500
8501         return NULL;
8502 }
8503
8504 static uint32_t
8505 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
8506 {
8507         uint32_t count = 0, i;
8508
8509         if (!label[0])
8510                 return 0;
8511
8512         for (i = 0; i < n; i++)
8513                 if (!strcmp(label, data[i].jmp_label))
8514                         count++;
8515
8516         return count;
8517 }
8518
8519 static int
8520 instr_label_check(struct instruction_data *instruction_data,
8521                   uint32_t n_instructions)
8522 {
8523         uint32_t i;
8524
8525         /* Check that all instruction labels are unique. */
8526         for (i = 0; i < n_instructions; i++) {
8527                 struct instruction_data *data = &instruction_data[i];
8528                 char *label = data->label;
8529                 uint32_t j;
8530
8531                 if (!label[0])
8532                         continue;
8533
8534                 for (j = i + 1; j < n_instructions; j++)
8535                         CHECK(strcmp(label, data[j].label), EINVAL);
8536         }
8537
8538         /* Get users for each instruction label. */
8539         for (i = 0; i < n_instructions; i++) {
8540                 struct instruction_data *data = &instruction_data[i];
8541                 char *label = data->label;
8542
8543                 data->n_users = label_is_used(instruction_data,
8544                                               n_instructions,
8545                                               label);
8546         }
8547
8548         return 0;
8549 }
8550
8551 static int
8552 instr_jmp_resolve(struct instruction *instructions,
8553                   struct instruction_data *instruction_data,
8554                   uint32_t n_instructions)
8555 {
8556         uint32_t i;
8557
8558         for (i = 0; i < n_instructions; i++) {
8559                 struct instruction *instr = &instructions[i];
8560                 struct instruction_data *data = &instruction_data[i];
8561                 struct instruction_data *found;
8562
8563                 if (!instruction_is_jmp(instr))
8564                         continue;
8565
8566                 found = label_find(instruction_data,
8567                                    n_instructions,
8568                                    data->jmp_label);
8569                 CHECK(found, EINVAL);
8570
8571                 instr->jmp.ip = &instructions[found - instruction_data];
8572         }
8573
8574         return 0;
8575 }
8576
8577 static int
8578 instr_verify(struct rte_swx_pipeline *p __rte_unused,
8579              struct action *a,
8580              struct instruction *instr,
8581              struct instruction_data *data __rte_unused,
8582              uint32_t n_instructions)
8583 {
8584         if (!a) {
8585                 enum instruction_type type;
8586                 uint32_t i;
8587
8588                 /* Check that the first instruction is rx. */
8589                 CHECK(instr[0].type == INSTR_RX, EINVAL);
8590
8591                 /* Check that there is at least one tx instruction. */
8592                 for (i = 0; i < n_instructions; i++) {
8593                         type = instr[i].type;
8594
8595                         if (instruction_is_tx(type))
8596                                 break;
8597                 }
8598                 CHECK(i < n_instructions, EINVAL);
8599
8600                 /* Check that the last instruction is either tx or unconditional
8601                  * jump.
8602                  */
8603                 type = instr[n_instructions - 1].type;
8604                 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
8605         }
8606
8607         if (a) {
8608                 enum instruction_type type;
8609                 uint32_t i;
8610
8611                 /* Check that there is at least one return or tx instruction. */
8612                 for (i = 0; i < n_instructions; i++) {
8613                         type = instr[i].type;
8614
8615                         if ((type == INSTR_RETURN) || instruction_is_tx(type))
8616                                 break;
8617                 }
8618                 CHECK(i < n_instructions, EINVAL);
8619         }
8620
8621         return 0;
8622 }
8623
8624 static uint32_t
8625 instr_compact(struct instruction *instructions,
8626               struct instruction_data *instruction_data,
8627               uint32_t n_instructions)
8628 {
8629         uint32_t i, pos = 0;
8630
8631         /* Eliminate the invalid instructions that have been optimized out. */
8632         for (i = 0; i < n_instructions; i++) {
8633                 struct instruction *instr = &instructions[i];
8634                 struct instruction_data *data = &instruction_data[i];
8635
8636                 if (data->invalid)
8637                         continue;
8638
8639                 if (i != pos) {
8640                         memcpy(&instructions[pos], instr, sizeof(*instr));
8641                         memcpy(&instruction_data[pos], data, sizeof(*data));
8642                 }
8643
8644                 pos++;
8645         }
8646
8647         return pos;
8648 }
8649
8650 static int
8651 instr_pattern_extract_many_search(struct instruction *instr,
8652                                   struct instruction_data *data,
8653                                   uint32_t n_instr,
8654                                   uint32_t *n_pattern_instr)
8655 {
8656         uint32_t i;
8657
8658         for (i = 0; i < n_instr; i++) {
8659                 if (data[i].invalid)
8660                         break;
8661
8662                 if (instr[i].type != INSTR_HDR_EXTRACT)
8663                         break;
8664
8665                 if (i == RTE_DIM(instr->io.hdr.header_id))
8666                         break;
8667
8668                 if (i && data[i].n_users)
8669                         break;
8670         }
8671
8672         if (i < 2)
8673                 return 0;
8674
8675         *n_pattern_instr = i;
8676         return 1;
8677 }
8678
8679 static void
8680 instr_pattern_extract_many_replace(struct instruction *instr,
8681                                    struct instruction_data *data,
8682                                    uint32_t n_instr)
8683 {
8684         uint32_t i;
8685
8686         for (i = 1; i < n_instr; i++) {
8687                 instr[0].type++;
8688                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8689                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8690                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8691
8692                 data[i].invalid = 1;
8693         }
8694 }
8695
8696 static uint32_t
8697 instr_pattern_extract_many_optimize(struct instruction *instructions,
8698                                     struct instruction_data *instruction_data,
8699                                     uint32_t n_instructions)
8700 {
8701         uint32_t i;
8702
8703         for (i = 0; i < n_instructions; ) {
8704                 struct instruction *instr = &instructions[i];
8705                 struct instruction_data *data = &instruction_data[i];
8706                 uint32_t n_instr = 0;
8707                 int detected;
8708
8709                 /* Extract many. */
8710                 detected = instr_pattern_extract_many_search(instr,
8711                                                              data,
8712                                                              n_instructions - i,
8713                                                              &n_instr);
8714                 if (detected) {
8715                         instr_pattern_extract_many_replace(instr,
8716                                                            data,
8717                                                            n_instr);
8718                         i += n_instr;
8719                         continue;
8720                 }
8721
8722                 /* No pattern starting at the current instruction. */
8723                 i++;
8724         }
8725
8726         /* Eliminate the invalid instructions that have been optimized out. */
8727         n_instructions = instr_compact(instructions,
8728                                        instruction_data,
8729                                        n_instructions);
8730
8731         return n_instructions;
8732 }
8733
8734 static int
8735 instr_pattern_emit_many_tx_search(struct instruction *instr,
8736                                   struct instruction_data *data,
8737                                   uint32_t n_instr,
8738                                   uint32_t *n_pattern_instr)
8739 {
8740         uint32_t i;
8741
8742         for (i = 0; i < n_instr; i++) {
8743                 if (data[i].invalid)
8744                         break;
8745
8746                 if (instr[i].type != INSTR_HDR_EMIT)
8747                         break;
8748
8749                 if (i == RTE_DIM(instr->io.hdr.header_id))
8750                         break;
8751
8752                 if (i && data[i].n_users)
8753                         break;
8754         }
8755
8756         if (!i)
8757                 return 0;
8758
8759         if (!instruction_is_tx(instr[i].type))
8760                 return 0;
8761
8762         if (data[i].n_users)
8763                 return 0;
8764
8765         i++;
8766
8767         *n_pattern_instr = i;
8768         return 1;
8769 }
8770
8771 static void
8772 instr_pattern_emit_many_tx_replace(struct instruction *instr,
8773                                    struct instruction_data *data,
8774                                    uint32_t n_instr)
8775 {
8776         uint32_t i;
8777
8778         /* Any emit instruction in addition to the first one. */
8779         for (i = 1; i < n_instr - 1; i++) {
8780                 instr[0].type++;
8781                 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8782                 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8783                 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8784
8785                 data[i].invalid = 1;
8786         }
8787
8788         /* The TX instruction is the last one in the pattern. */
8789         instr[0].type++;
8790         instr[0].io.io.offset = instr[i].io.io.offset;
8791         instr[0].io.io.n_bits = instr[i].io.io.n_bits;
8792         data[i].invalid = 1;
8793 }
8794
8795 static uint32_t
8796 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
8797                                     struct instruction_data *instruction_data,
8798                                     uint32_t n_instructions)
8799 {
8800         uint32_t i;
8801
8802         for (i = 0; i < n_instructions; ) {
8803                 struct instruction *instr = &instructions[i];
8804                 struct instruction_data *data = &instruction_data[i];
8805                 uint32_t n_instr = 0;
8806                 int detected;
8807
8808                 /* Emit many + TX. */
8809                 detected = instr_pattern_emit_many_tx_search(instr,
8810                                                              data,
8811                                                              n_instructions - i,
8812                                                              &n_instr);
8813                 if (detected) {
8814                         instr_pattern_emit_many_tx_replace(instr,
8815                                                            data,
8816                                                            n_instr);
8817                         i += n_instr;
8818                         continue;
8819                 }
8820
8821                 /* No pattern starting at the current instruction. */
8822                 i++;
8823         }
8824
8825         /* Eliminate the invalid instructions that have been optimized out. */
8826         n_instructions = instr_compact(instructions,
8827                                        instruction_data,
8828                                        n_instructions);
8829
8830         return n_instructions;
8831 }
8832
8833 static uint32_t
8834 action_arg_src_mov_count(struct action *a,
8835                          uint32_t arg_id,
8836                          struct instruction *instructions,
8837                          struct instruction_data *instruction_data,
8838                          uint32_t n_instructions);
8839
8840 static int
8841 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
8842                                       struct action *a,
8843                                       struct instruction *instr,
8844                                       struct instruction_data *data,
8845                                       uint32_t n_instr,
8846                                       struct instruction *instructions,
8847                                       struct instruction_data *instruction_data,
8848                                       uint32_t n_instructions,
8849                                       uint32_t *n_pattern_instr)
8850 {
8851         struct header *h;
8852         uint32_t src_field_id, i, j;
8853
8854         /* Prerequisites. */
8855         if (!a || !a->st)
8856                 return 0;
8857
8858         /* First instruction: MOV_HM. */
8859         if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
8860                 return 0;
8861
8862         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
8863         if (!h || h->st->var_size)
8864                 return 0;
8865
8866         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
8867                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
8868                         break;
8869
8870         if (src_field_id == a->st->n_fields)
8871                 return 0;
8872
8873         if (instr[0].mov.dst.offset ||
8874             (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
8875             instr[0].mov.src.struct_id ||
8876             (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
8877             (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
8878                 return 0;
8879
8880         if ((n_instr < h->st->n_fields + 1) ||
8881              (a->st->n_fields < src_field_id + h->st->n_fields + 1))
8882                 return 0;
8883
8884         /* Subsequent instructions: MOV_HM. */
8885         for (i = 1; i < h->st->n_fields; i++)
8886                 if (data[i].invalid ||
8887                     data[i].n_users ||
8888                     (instr[i].type != INSTR_MOV_HM) ||
8889                     (instr[i].mov.dst.struct_id != h->struct_id) ||
8890                     (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
8891                     (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
8892                     instr[i].mov.src.struct_id ||
8893                     (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
8894                     (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
8895                     (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
8896                         return 0;
8897
8898         /* Last instruction: HDR_VALIDATE. */
8899         if ((instr[i].type != INSTR_HDR_VALIDATE) ||
8900             (instr[i].valid.header_id != h->id))
8901                 return 0;
8902
8903         /* Check that none of the action args that are used as source for this
8904          * DMA transfer are not used as source in any other mov instruction.
8905          */
8906         for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
8907                 uint32_t n_users;
8908
8909                 n_users = action_arg_src_mov_count(a,
8910                                                    j,
8911                                                    instructions,
8912                                                    instruction_data,
8913                                                    n_instructions);
8914                 if (n_users > 1)
8915                         return 0;
8916         }
8917
8918         *n_pattern_instr = 1 + i;
8919         return 1;
8920 }
8921
8922 static void
8923 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
8924                                        struct action *a,
8925                                        struct instruction *instr,
8926                                        struct instruction_data *data,
8927                                        uint32_t n_instr)
8928 {
8929         struct header *h;
8930         uint32_t src_field_id, src_offset, i;
8931
8932         /* Read from the instructions before they are modified. */
8933         h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
8934         if (!h)
8935                 return;
8936
8937         for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
8938                 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
8939                         break;
8940
8941         if (src_field_id == a->st->n_fields)
8942                 return;
8943
8944         src_offset = instr[0].mov.src.offset;
8945
8946         /* Modify the instructions. */
8947         instr[0].type = INSTR_DMA_HT;
8948         instr[0].dma.dst.header_id[0] = h->id;
8949         instr[0].dma.dst.struct_id[0] = h->struct_id;
8950         instr[0].dma.src.offset[0] = (uint8_t)src_offset;
8951         instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
8952
8953         for (i = 1; i < n_instr; i++)
8954                 data[i].invalid = 1;
8955
8956         /* Update the endianness of the action arguments to header endianness. */
8957         for (i = 0; i < h->st->n_fields; i++)
8958                 a->args_endianness[src_field_id + i] = 1;
8959 }
8960
8961 static uint32_t
8962 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
8963                                         struct action *a,
8964                                         struct instruction *instructions,
8965                                         struct instruction_data *instruction_data,
8966                                         uint32_t n_instructions)
8967 {
8968         uint32_t i;
8969
8970         if (!a || !a->st)
8971                 return n_instructions;
8972
8973         for (i = 0; i < n_instructions; ) {
8974                 struct instruction *instr = &instructions[i];
8975                 struct instruction_data *data = &instruction_data[i];
8976                 uint32_t n_instr = 0;
8977                 int detected;
8978
8979                 /* Mov all + validate. */
8980                 detected = instr_pattern_mov_all_validate_search(p,
8981                                                                  a,
8982                                                                  instr,
8983                                                                  data,
8984                                                                  n_instructions - i,
8985                                                                  instructions,
8986                                                                  instruction_data,
8987                                                                  n_instructions,
8988                                                                  &n_instr);
8989                 if (detected) {
8990                         instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
8991                         i += n_instr;
8992                         continue;
8993                 }
8994
8995                 /* No pattern starting at the current instruction. */
8996                 i++;
8997         }
8998
8999         /* Eliminate the invalid instructions that have been optimized out. */
9000         n_instructions = instr_compact(instructions,
9001                                        instruction_data,
9002                                        n_instructions);
9003
9004         return n_instructions;
9005 }
9006
9007 static int
9008 instr_pattern_dma_many_search(struct instruction *instr,
9009                               struct instruction_data *data,
9010                               uint32_t n_instr,
9011                               uint32_t *n_pattern_instr)
9012 {
9013         uint32_t i;
9014
9015         for (i = 0; i < n_instr; i++) {
9016                 if (data[i].invalid)
9017                         break;
9018
9019                 if (instr[i].type != INSTR_DMA_HT)
9020                         break;
9021
9022                 if (i == RTE_DIM(instr->dma.dst.header_id))
9023                         break;
9024
9025                 if (i && data[i].n_users)
9026                         break;
9027         }
9028
9029         if (i < 2)
9030                 return 0;
9031
9032         *n_pattern_instr = i;
9033         return 1;
9034 }
9035
9036 static void
9037 instr_pattern_dma_many_replace(struct instruction *instr,
9038                                struct instruction_data *data,
9039                                uint32_t n_instr)
9040 {
9041         uint32_t i;
9042
9043         for (i = 1; i < n_instr; i++) {
9044                 instr[0].type++;
9045                 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
9046                 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
9047                 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
9048                 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
9049
9050                 data[i].invalid = 1;
9051         }
9052 }
9053
9054 static uint32_t
9055 instr_pattern_dma_many_optimize(struct instruction *instructions,
9056                struct instruction_data *instruction_data,
9057                uint32_t n_instructions)
9058 {
9059         uint32_t i;
9060
9061         for (i = 0; i < n_instructions; ) {
9062                 struct instruction *instr = &instructions[i];
9063                 struct instruction_data *data = &instruction_data[i];
9064                 uint32_t n_instr = 0;
9065                 int detected;
9066
9067                 /* DMA many. */
9068                 detected = instr_pattern_dma_many_search(instr,
9069                                                          data,
9070                                                          n_instructions - i,
9071                                                          &n_instr);
9072                 if (detected) {
9073                         instr_pattern_dma_many_replace(instr, data, n_instr);
9074                         i += n_instr;
9075                         continue;
9076                 }
9077
9078                 /* No pattern starting at the current instruction. */
9079                 i++;
9080         }
9081
9082         /* Eliminate the invalid instructions that have been optimized out. */
9083         n_instructions = instr_compact(instructions,
9084                                        instruction_data,
9085                                        n_instructions);
9086
9087         return n_instructions;
9088 }
9089
9090 static uint32_t
9091 instr_optimize(struct rte_swx_pipeline *p,
9092                struct action *a,
9093                struct instruction *instructions,
9094                struct instruction_data *instruction_data,
9095                uint32_t n_instructions)
9096 {
9097         /* Extract many. */
9098         n_instructions = instr_pattern_extract_many_optimize(instructions,
9099                                                              instruction_data,
9100                                                              n_instructions);
9101
9102         /* Emit many + TX. */
9103         n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
9104                                                              instruction_data,
9105                                                              n_instructions);
9106
9107         /* Mov all + validate. */
9108         n_instructions = instr_pattern_mov_all_validate_optimize(p,
9109                                                                  a,
9110                                                                  instructions,
9111                                                                  instruction_data,
9112                                                                  n_instructions);
9113
9114         /* DMA many. */
9115         n_instructions = instr_pattern_dma_many_optimize(instructions,
9116                                                          instruction_data,
9117                                                          n_instructions);
9118
9119         return n_instructions;
9120 }
9121
9122 static int
9123 instruction_config(struct rte_swx_pipeline *p,
9124                    struct action *a,
9125                    const char **instructions,
9126                    uint32_t n_instructions)
9127 {
9128         struct instruction *instr = NULL;
9129         struct instruction_data *data = NULL;
9130         int err = 0;
9131         uint32_t i;
9132
9133         CHECK(n_instructions, EINVAL);
9134         CHECK(instructions, EINVAL);
9135         for (i = 0; i < n_instructions; i++)
9136                 CHECK_INSTRUCTION(instructions[i], EINVAL);
9137
9138         /* Memory allocation. */
9139         instr = calloc(n_instructions, sizeof(struct instruction));
9140         if (!instr) {
9141                 err = -ENOMEM;
9142                 goto error;
9143         }
9144
9145         data = calloc(n_instructions, sizeof(struct instruction_data));
9146         if (!data) {
9147                 err = -ENOMEM;
9148                 goto error;
9149         }
9150
9151         for (i = 0; i < n_instructions; i++) {
9152                 char *string = strdup(instructions[i]);
9153                 if (!string) {
9154                         err = -ENOMEM;
9155                         goto error;
9156                 }
9157
9158                 err = instr_translate(p, a, string, &instr[i], &data[i]);
9159                 if (err) {
9160                         free(string);
9161                         goto error;
9162                 }
9163
9164                 free(string);
9165         }
9166
9167         err = instr_label_check(data, n_instructions);
9168         if (err)
9169                 goto error;
9170
9171         err = instr_verify(p, a, instr, data, n_instructions);
9172         if (err)
9173                 goto error;
9174
9175         n_instructions = instr_optimize(p, a, instr, data, n_instructions);
9176
9177         err = instr_jmp_resolve(instr, data, n_instructions);
9178         if (err)
9179                 goto error;
9180
9181         if (a) {
9182                 a->instructions = instr;
9183                 a->n_instructions = n_instructions;
9184         } else {
9185                 p->instructions = instr;
9186                 p->n_instructions = n_instructions;
9187         }
9188
9189         free(data);
9190         return 0;
9191
9192 error:
9193         free(data);
9194         free(instr);
9195         return err;
9196 }
9197
9198 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
9199
9200 static instr_exec_t instruction_table[] = {
9201         [INSTR_RX] = instr_rx_exec,
9202         [INSTR_TX] = instr_tx_exec,
9203         [INSTR_TX_I] = instr_tx_i_exec,
9204
9205         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
9206         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
9207         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
9208         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
9209         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
9210         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
9211         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
9212         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
9213         [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
9214         [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
9215
9216         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
9217         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
9218         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
9219         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
9220         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
9221         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
9222         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
9223         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
9224         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
9225
9226         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
9227         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
9228
9229         [INSTR_MOV] = instr_mov_exec,
9230         [INSTR_MOV_MH] = instr_mov_mh_exec,
9231         [INSTR_MOV_HM] = instr_mov_hm_exec,
9232         [INSTR_MOV_HH] = instr_mov_hh_exec,
9233         [INSTR_MOV_I] = instr_mov_i_exec,
9234
9235         [INSTR_DMA_HT] = instr_dma_ht_exec,
9236         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
9237         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
9238         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
9239         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
9240         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
9241         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
9242         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
9243
9244         [INSTR_ALU_ADD] = instr_alu_add_exec,
9245         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
9246         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
9247         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
9248         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
9249         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
9250
9251         [INSTR_ALU_SUB] = instr_alu_sub_exec,
9252         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
9253         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
9254         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
9255         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
9256         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
9257
9258         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
9259         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
9260         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
9261         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
9262
9263         [INSTR_ALU_AND] = instr_alu_and_exec,
9264         [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
9265         [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
9266         [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
9267         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
9268
9269         [INSTR_ALU_OR] = instr_alu_or_exec,
9270         [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
9271         [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
9272         [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
9273         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
9274
9275         [INSTR_ALU_XOR] = instr_alu_xor_exec,
9276         [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
9277         [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
9278         [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
9279         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
9280
9281         [INSTR_ALU_SHL] = instr_alu_shl_exec,
9282         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
9283         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
9284         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
9285         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
9286         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
9287
9288         [INSTR_ALU_SHR] = instr_alu_shr_exec,
9289         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
9290         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
9291         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
9292         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
9293         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
9294
9295         [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
9296         [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
9297         [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
9298
9299         [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
9300         [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
9301         [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
9302         [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
9303         [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
9304         [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
9305
9306         [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
9307         [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
9308         [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
9309         [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
9310         [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
9311         [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
9312         [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
9313         [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
9314         [INSTR_REGWR_RII] = instr_regwr_rii_exec,
9315
9316         [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
9317         [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
9318         [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
9319         [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
9320         [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
9321         [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
9322         [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
9323         [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
9324         [INSTR_REGADD_RII] = instr_regadd_rii_exec,
9325
9326         [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
9327         [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
9328         [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
9329
9330         [INSTR_METER_HHM] = instr_meter_hhm_exec,
9331         [INSTR_METER_HHI] = instr_meter_hhi_exec,
9332         [INSTR_METER_HMM] = instr_meter_hmm_exec,
9333         [INSTR_METER_HMI] = instr_meter_hmi_exec,
9334         [INSTR_METER_MHM] = instr_meter_mhm_exec,
9335         [INSTR_METER_MHI] = instr_meter_mhi_exec,
9336         [INSTR_METER_MMM] = instr_meter_mmm_exec,
9337         [INSTR_METER_MMI] = instr_meter_mmi_exec,
9338         [INSTR_METER_IHM] = instr_meter_ihm_exec,
9339         [INSTR_METER_IHI] = instr_meter_ihi_exec,
9340         [INSTR_METER_IMM] = instr_meter_imm_exec,
9341         [INSTR_METER_IMI] = instr_meter_imi_exec,
9342
9343         [INSTR_TABLE] = instr_table_exec,
9344         [INSTR_SELECTOR] = instr_selector_exec,
9345         [INSTR_LEARNER] = instr_learner_exec,
9346         [INSTR_LEARNER_LEARN] = instr_learn_exec,
9347         [INSTR_LEARNER_FORGET] = instr_forget_exec,
9348         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
9349         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
9350
9351         [INSTR_JMP] = instr_jmp_exec,
9352         [INSTR_JMP_VALID] = instr_jmp_valid_exec,
9353         [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
9354         [INSTR_JMP_HIT] = instr_jmp_hit_exec,
9355         [INSTR_JMP_MISS] = instr_jmp_miss_exec,
9356         [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
9357         [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
9358
9359         [INSTR_JMP_EQ] = instr_jmp_eq_exec,
9360         [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
9361         [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
9362         [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
9363         [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
9364
9365         [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
9366         [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
9367         [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
9368         [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
9369         [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
9370
9371         [INSTR_JMP_LT] = instr_jmp_lt_exec,
9372         [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
9373         [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
9374         [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
9375         [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
9376         [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
9377
9378         [INSTR_JMP_GT] = instr_jmp_gt_exec,
9379         [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
9380         [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
9381         [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
9382         [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
9383         [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
9384
9385         [INSTR_RETURN] = instr_return_exec,
9386 };
9387
9388 static inline void
9389 instr_exec(struct rte_swx_pipeline *p)
9390 {
9391         struct thread *t = &p->threads[p->thread_id];
9392         struct instruction *ip = t->ip;
9393         instr_exec_t instr = instruction_table[ip->type];
9394
9395         instr(p);
9396 }
9397
9398 /*
9399  * Action.
9400  */
9401 static struct action *
9402 action_find(struct rte_swx_pipeline *p, const char *name)
9403 {
9404         struct action *elem;
9405
9406         if (!name)
9407                 return NULL;
9408
9409         TAILQ_FOREACH(elem, &p->actions, node)
9410                 if (strcmp(elem->name, name) == 0)
9411                         return elem;
9412
9413         return NULL;
9414 }
9415
9416 static struct action *
9417 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9418 {
9419         struct action *action = NULL;
9420
9421         TAILQ_FOREACH(action, &p->actions, node)
9422                 if (action->id == id)
9423                         return action;
9424
9425         return NULL;
9426 }
9427
9428 static struct field *
9429 action_field_find(struct action *a, const char *name)
9430 {
9431         return a->st ? struct_type_field_find(a->st, name) : NULL;
9432 }
9433
9434 static struct field *
9435 action_field_parse(struct action *action, const char *name)
9436 {
9437         if (name[0] != 't' || name[1] != '.')
9438                 return NULL;
9439
9440         return action_field_find(action, &name[2]);
9441 }
9442
9443 static int
9444 action_has_nbo_args(struct action *a)
9445 {
9446         uint32_t i;
9447
9448         /* Return if the action does not have any args. */
9449         if (!a->st)
9450                 return 0; /* FALSE */
9451
9452         for (i = 0; i < a->st->n_fields; i++)
9453                 if (a->args_endianness[i])
9454                         return 1; /* TRUE */
9455
9456         return 0; /* FALSE */
9457 }
9458
9459 static int
9460 action_does_learning(struct action *a)
9461 {
9462         uint32_t i;
9463
9464         for (i = 0; i < a->n_instructions; i++)
9465                 switch (a->instructions[i].type) {
9466                 case INSTR_LEARNER_LEARN:
9467                         return 1; /* TRUE */
9468
9469                 case INSTR_LEARNER_FORGET:
9470                         return 1; /* TRUE */
9471
9472                 default:
9473                         continue;
9474                 }
9475
9476         return 0; /* FALSE */
9477 }
9478
9479 int
9480 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
9481                                const char *name,
9482                                const char *args_struct_type_name,
9483                                const char **instructions,
9484                                uint32_t n_instructions)
9485 {
9486         struct struct_type *args_struct_type = NULL;
9487         struct action *a;
9488         int err;
9489
9490         CHECK(p, EINVAL);
9491
9492         CHECK_NAME(name, EINVAL);
9493         CHECK(!action_find(p, name), EEXIST);
9494
9495         if (args_struct_type_name) {
9496                 CHECK_NAME(args_struct_type_name, EINVAL);
9497                 args_struct_type = struct_type_find(p, args_struct_type_name);
9498                 CHECK(args_struct_type, EINVAL);
9499                 CHECK(!args_struct_type->var_size, EINVAL);
9500         }
9501
9502         /* Node allocation. */
9503         a = calloc(1, sizeof(struct action));
9504         CHECK(a, ENOMEM);
9505         if (args_struct_type) {
9506                 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
9507                 if (!a->args_endianness) {
9508                         free(a);
9509                         CHECK(0, ENOMEM);
9510                 }
9511         }
9512
9513         /* Node initialization. */
9514         strcpy(a->name, name);
9515         a->st = args_struct_type;
9516         a->id = p->n_actions;
9517
9518         /* Instruction translation. */
9519         err = instruction_config(p, a, instructions, n_instructions);
9520         if (err) {
9521                 free(a->args_endianness);
9522                 free(a);
9523                 return err;
9524         }
9525
9526         /* Node add to tailq. */
9527         TAILQ_INSERT_TAIL(&p->actions, a, node);
9528         p->n_actions++;
9529
9530         return 0;
9531 }
9532
9533 static int
9534 action_build(struct rte_swx_pipeline *p)
9535 {
9536         struct action *action;
9537
9538         p->action_instructions = calloc(p->n_actions,
9539                                         sizeof(struct instruction *));
9540         CHECK(p->action_instructions, ENOMEM);
9541
9542         TAILQ_FOREACH(action, &p->actions, node)
9543                 p->action_instructions[action->id] = action->instructions;
9544
9545         return 0;
9546 }
9547
9548 static void
9549 action_build_free(struct rte_swx_pipeline *p)
9550 {
9551         free(p->action_instructions);
9552         p->action_instructions = NULL;
9553 }
9554
9555 static void
9556 action_free(struct rte_swx_pipeline *p)
9557 {
9558         action_build_free(p);
9559
9560         for ( ; ; ) {
9561                 struct action *action;
9562
9563                 action = TAILQ_FIRST(&p->actions);
9564                 if (!action)
9565                         break;
9566
9567                 TAILQ_REMOVE(&p->actions, action, node);
9568                 free(action->instructions);
9569                 free(action);
9570         }
9571 }
9572
9573 static uint32_t
9574 action_arg_src_mov_count(struct action *a,
9575                          uint32_t arg_id,
9576                          struct instruction *instructions,
9577                          struct instruction_data *instruction_data,
9578                          uint32_t n_instructions)
9579 {
9580         uint32_t offset, n_users = 0, i;
9581
9582         if (!a->st ||
9583             (arg_id >= a->st->n_fields) ||
9584             !instructions ||
9585             !instruction_data ||
9586             !n_instructions)
9587                 return 0;
9588
9589         offset = a->st->fields[arg_id].offset / 8;
9590
9591         for (i = 0; i < n_instructions; i++) {
9592                 struct instruction *instr = &instructions[i];
9593                 struct instruction_data *data = &instruction_data[i];
9594
9595                 if (data->invalid ||
9596                     ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
9597                     instr->mov.src.struct_id ||
9598                     (instr->mov.src.offset != offset))
9599                         continue;
9600
9601                 n_users++;
9602         }
9603
9604         return n_users;
9605 }
9606
9607 /*
9608  * Table.
9609  */
9610 static struct table_type *
9611 table_type_find(struct rte_swx_pipeline *p, const char *name)
9612 {
9613         struct table_type *elem;
9614
9615         TAILQ_FOREACH(elem, &p->table_types, node)
9616                 if (strcmp(elem->name, name) == 0)
9617                         return elem;
9618
9619         return NULL;
9620 }
9621
9622 static struct table_type *
9623 table_type_resolve(struct rte_swx_pipeline *p,
9624                    const char *recommended_type_name,
9625                    enum rte_swx_table_match_type match_type)
9626 {
9627         struct table_type *elem;
9628
9629         /* Only consider the recommended type if the match type is correct. */
9630         if (recommended_type_name)
9631                 TAILQ_FOREACH(elem, &p->table_types, node)
9632                         if (!strcmp(elem->name, recommended_type_name) &&
9633                             (elem->match_type == match_type))
9634                                 return elem;
9635
9636         /* Ignore the recommended type and get the first element with this match
9637          * type.
9638          */
9639         TAILQ_FOREACH(elem, &p->table_types, node)
9640                 if (elem->match_type == match_type)
9641                         return elem;
9642
9643         return NULL;
9644 }
9645
9646 static struct table *
9647 table_find(struct rte_swx_pipeline *p, const char *name)
9648 {
9649         struct table *elem;
9650
9651         TAILQ_FOREACH(elem, &p->tables, node)
9652                 if (strcmp(elem->name, name) == 0)
9653                         return elem;
9654
9655         return NULL;
9656 }
9657
9658 static struct table *
9659 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9660 {
9661         struct table *table = NULL;
9662
9663         TAILQ_FOREACH(table, &p->tables, node)
9664                 if (table->id == id)
9665                         return table;
9666
9667         return NULL;
9668 }
9669
9670 int
9671 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
9672                                      const char *name,
9673                                      enum rte_swx_table_match_type match_type,
9674                                      struct rte_swx_table_ops *ops)
9675 {
9676         struct table_type *elem;
9677
9678         CHECK(p, EINVAL);
9679
9680         CHECK_NAME(name, EINVAL);
9681         CHECK(!table_type_find(p, name), EEXIST);
9682
9683         CHECK(ops, EINVAL);
9684         CHECK(ops->create, EINVAL);
9685         CHECK(ops->lkp, EINVAL);
9686         CHECK(ops->free, EINVAL);
9687
9688         /* Node allocation. */
9689         elem = calloc(1, sizeof(struct table_type));
9690         CHECK(elem, ENOMEM);
9691
9692         /* Node initialization. */
9693         strcpy(elem->name, name);
9694         elem->match_type = match_type;
9695         memcpy(&elem->ops, ops, sizeof(*ops));
9696
9697         /* Node add to tailq. */
9698         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
9699
9700         return 0;
9701 }
9702
9703 static int
9704 table_match_type_resolve(struct rte_swx_match_field_params *fields,
9705                          uint32_t n_fields,
9706                          enum rte_swx_table_match_type *match_type)
9707 {
9708         uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
9709
9710         for (i = 0; i < n_fields; i++) {
9711                 struct rte_swx_match_field_params  *f = &fields[i];
9712
9713                 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
9714                         n_fields_em++;
9715
9716                 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
9717                         n_fields_lpm++;
9718         }
9719
9720         if ((n_fields_lpm > 1) ||
9721             (n_fields_lpm && (n_fields_em != n_fields - 1)))
9722                 return -EINVAL;
9723
9724         *match_type = (n_fields_em == n_fields) ?
9725                        RTE_SWX_TABLE_MATCH_EXACT :
9726                        RTE_SWX_TABLE_MATCH_WILDCARD;
9727
9728         return 0;
9729 }
9730
9731 static int
9732 table_match_fields_check(struct rte_swx_pipeline *p,
9733                          struct rte_swx_pipeline_table_params *params,
9734                          struct header **header)
9735 {
9736         struct header *h0 = NULL;
9737         struct field *hf, *mf;
9738         uint32_t *offset = NULL, i;
9739         int status = 0;
9740
9741         /* Return if no match fields. */
9742         if (!params->n_fields) {
9743                 if (params->fields) {
9744                         status = -EINVAL;
9745                         goto end;
9746                 }
9747
9748                 if (header)
9749                         *header = NULL;
9750
9751                 return 0;
9752         }
9753
9754         /* Memory allocation. */
9755         offset = calloc(params->n_fields, sizeof(uint32_t));
9756         if (!offset) {
9757                 status = -ENOMEM;
9758                 goto end;
9759         }
9760
9761         /* Check that all the match fields belong to either the same header or
9762          * to the meta-data.
9763          */
9764         hf = header_field_parse(p, params->fields[0].name, &h0);
9765         mf = metadata_field_parse(p, params->fields[0].name);
9766         if ((!hf && !mf) || (hf && hf->var_size)) {
9767                 status = -EINVAL;
9768                 goto end;
9769         }
9770
9771         offset[0] = h0 ? hf->offset : mf->offset;
9772
9773         for (i = 1; i < params->n_fields; i++)
9774                 if (h0) {
9775                         struct header *h;
9776
9777                         hf = header_field_parse(p, params->fields[i].name, &h);
9778                         if (!hf || (h->id != h0->id) || hf->var_size) {
9779                                 status = -EINVAL;
9780                                 goto end;
9781                         }
9782
9783                         offset[i] = hf->offset;
9784                 } else {
9785                         mf = metadata_field_parse(p, params->fields[i].name);
9786                         if (!mf) {
9787                                 status = -EINVAL;
9788                                 goto end;
9789                         }
9790
9791                         offset[i] = mf->offset;
9792                 }
9793
9794         /* Check that there are no duplicated match fields. */
9795         for (i = 0; i < params->n_fields; i++) {
9796                 uint32_t j;
9797
9798                 for (j = 0; j < i; j++)
9799                         if (offset[j] == offset[i]) {
9800                                 status = -EINVAL;
9801                                 goto end;
9802                         }
9803         }
9804
9805         /* Return. */
9806         if (header)
9807                 *header = h0;
9808
9809 end:
9810         free(offset);
9811         return status;
9812 }
9813
9814 int
9815 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
9816                               const char *name,
9817                               struct rte_swx_pipeline_table_params *params,
9818                               const char *recommended_table_type_name,
9819                               const char *args,
9820                               uint32_t size)
9821 {
9822         struct table_type *type;
9823         struct table *t;
9824         struct action *default_action;
9825         struct header *header = NULL;
9826         uint32_t action_data_size_max = 0, i;
9827         int status = 0;
9828
9829         CHECK(p, EINVAL);
9830
9831         CHECK_NAME(name, EINVAL);
9832         CHECK(!table_find(p, name), EEXIST);
9833         CHECK(!selector_find(p, name), EEXIST);
9834         CHECK(!learner_find(p, name), EEXIST);
9835
9836         CHECK(params, EINVAL);
9837
9838         /* Match checks. */
9839         status = table_match_fields_check(p, params, &header);
9840         if (status)
9841                 return status;
9842
9843         /* Action checks. */
9844         CHECK(params->n_actions, EINVAL);
9845         CHECK(params->action_names, EINVAL);
9846         for (i = 0; i < params->n_actions; i++) {
9847                 const char *action_name = params->action_names[i];
9848                 struct action *a;
9849                 uint32_t action_data_size;
9850
9851                 CHECK_NAME(action_name, EINVAL);
9852
9853                 a = action_find(p, action_name);
9854                 CHECK(a, EINVAL);
9855                 CHECK(!action_does_learning(a), EINVAL);
9856
9857                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
9858                 if (action_data_size > action_data_size_max)
9859                         action_data_size_max = action_data_size;
9860         }
9861
9862         CHECK_NAME(params->default_action_name, EINVAL);
9863         for (i = 0; i < p->n_actions; i++)
9864                 if (!strcmp(params->action_names[i],
9865                             params->default_action_name))
9866                         break;
9867         CHECK(i < params->n_actions, EINVAL);
9868         default_action = action_find(p, params->default_action_name);
9869         CHECK((default_action->st && params->default_action_data) ||
9870               !params->default_action_data, EINVAL);
9871
9872         /* Table type checks. */
9873         if (recommended_table_type_name)
9874                 CHECK_NAME(recommended_table_type_name, EINVAL);
9875
9876         if (params->n_fields) {
9877                 enum rte_swx_table_match_type match_type;
9878
9879                 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
9880                 if (status)
9881                         return status;
9882
9883                 type = table_type_resolve(p, recommended_table_type_name, match_type);
9884                 CHECK(type, EINVAL);
9885         } else {
9886                 type = NULL;
9887         }
9888
9889         /* Memory allocation. */
9890         t = calloc(1, sizeof(struct table));
9891         CHECK(t, ENOMEM);
9892
9893         t->fields = calloc(params->n_fields, sizeof(struct match_field));
9894         if (!t->fields) {
9895                 free(t);
9896                 CHECK(0, ENOMEM);
9897         }
9898
9899         t->actions = calloc(params->n_actions, sizeof(struct action *));
9900         if (!t->actions) {
9901                 free(t->fields);
9902                 free(t);
9903                 CHECK(0, ENOMEM);
9904         }
9905
9906         if (action_data_size_max) {
9907                 t->default_action_data = calloc(1, action_data_size_max);
9908                 if (!t->default_action_data) {
9909                         free(t->actions);
9910                         free(t->fields);
9911                         free(t);
9912                         CHECK(0, ENOMEM);
9913                 }
9914         }
9915
9916         /* Node initialization. */
9917         strcpy(t->name, name);
9918         if (args && args[0])
9919                 strcpy(t->args, args);
9920         t->type = type;
9921
9922         for (i = 0; i < params->n_fields; i++) {
9923                 struct rte_swx_match_field_params *field = &params->fields[i];
9924                 struct match_field *f = &t->fields[i];
9925
9926                 f->match_type = field->match_type;
9927                 f->field = header ?
9928                         header_field_parse(p, field->name, NULL) :
9929                         metadata_field_parse(p, field->name);
9930         }
9931         t->n_fields = params->n_fields;
9932         t->header = header;
9933
9934         for (i = 0; i < params->n_actions; i++)
9935                 t->actions[i] = action_find(p, params->action_names[i]);
9936         t->default_action = default_action;
9937         if (default_action->st)
9938                 memcpy(t->default_action_data,
9939                        params->default_action_data,
9940                        default_action->st->n_bits / 8);
9941         t->n_actions = params->n_actions;
9942         t->default_action_is_const = params->default_action_is_const;
9943         t->action_data_size_max = action_data_size_max;
9944
9945         t->size = size;
9946         t->id = p->n_tables;
9947
9948         /* Node add to tailq. */
9949         TAILQ_INSERT_TAIL(&p->tables, t, node);
9950         p->n_tables++;
9951
9952         return 0;
9953 }
9954
9955 static struct rte_swx_table_params *
9956 table_params_get(struct table *table)
9957 {
9958         struct rte_swx_table_params *params;
9959         struct field *first, *last;
9960         uint8_t *key_mask;
9961         uint32_t key_size, key_offset, action_data_size, i;
9962
9963         /* Memory allocation. */
9964         params = calloc(1, sizeof(struct rte_swx_table_params));
9965         if (!params)
9966                 return NULL;
9967
9968         /* Find first (smallest offset) and last (biggest offset) match fields. */
9969         first = table->fields[0].field;
9970         last = table->fields[0].field;
9971
9972         for (i = 0; i < table->n_fields; i++) {
9973                 struct field *f = table->fields[i].field;
9974
9975                 if (f->offset < first->offset)
9976                         first = f;
9977
9978                 if (f->offset > last->offset)
9979                         last = f;
9980         }
9981
9982         /* Key offset and size. */
9983         key_offset = first->offset / 8;
9984         key_size = (last->offset + last->n_bits - first->offset) / 8;
9985
9986         /* Memory allocation. */
9987         key_mask = calloc(1, key_size);
9988         if (!key_mask) {
9989                 free(params);
9990                 return NULL;
9991         }
9992
9993         /* Key mask. */
9994         for (i = 0; i < table->n_fields; i++) {
9995                 struct field *f = table->fields[i].field;
9996                 uint32_t start = (f->offset - first->offset) / 8;
9997                 size_t size = f->n_bits / 8;
9998
9999                 memset(&key_mask[start], 0xFF, size);
10000         }
10001
10002         /* Action data size. */
10003         action_data_size = 0;
10004         for (i = 0; i < table->n_actions; i++) {
10005                 struct action *action = table->actions[i];
10006                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
10007
10008                 if (ads > action_data_size)
10009                         action_data_size = ads;
10010         }
10011
10012         /* Fill in. */
10013         params->match_type = table->type->match_type;
10014         params->key_size = key_size;
10015         params->key_offset = key_offset;
10016         params->key_mask0 = key_mask;
10017         params->action_data_size = action_data_size;
10018         params->n_keys_max = table->size;
10019
10020         return params;
10021 }
10022
10023 static void
10024 table_params_free(struct rte_swx_table_params *params)
10025 {
10026         if (!params)
10027                 return;
10028
10029         free(params->key_mask0);
10030         free(params);
10031 }
10032
10033 static int
10034 table_stub_lkp(void *table __rte_unused,
10035                void *mailbox __rte_unused,
10036                uint8_t **key __rte_unused,
10037                uint64_t *action_id __rte_unused,
10038                uint8_t **action_data __rte_unused,
10039                int *hit)
10040 {
10041         *hit = 0;
10042         return 1; /* DONE. */
10043 }
10044
10045 static int
10046 table_build(struct rte_swx_pipeline *p)
10047 {
10048         uint32_t i;
10049
10050         /* Per pipeline: table statistics. */
10051         p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
10052         CHECK(p->table_stats, ENOMEM);
10053
10054         for (i = 0; i < p->n_tables; i++) {
10055                 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
10056                 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
10057         }
10058
10059         /* Per thread: table runt-time. */
10060         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10061                 struct thread *t = &p->threads[i];
10062                 struct table *table;
10063
10064                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
10065                 CHECK(t->tables, ENOMEM);
10066
10067                 TAILQ_FOREACH(table, &p->tables, node) {
10068                         struct table_runtime *r = &t->tables[table->id];
10069
10070                         if (table->type) {
10071                                 uint64_t size;
10072
10073                                 size = table->type->ops.mailbox_size_get();
10074
10075                                 /* r->func. */
10076                                 r->func = table->type->ops.lkp;
10077
10078                                 /* r->mailbox. */
10079                                 if (size) {
10080                                         r->mailbox = calloc(1, size);
10081                                         CHECK(r->mailbox, ENOMEM);
10082                                 }
10083
10084                                 /* r->key. */
10085                                 r->key = table->header ?
10086                                         &t->structs[table->header->struct_id] :
10087                                         &t->structs[p->metadata_struct_id];
10088                         } else {
10089                                 r->func = table_stub_lkp;
10090                         }
10091                 }
10092         }
10093
10094         return 0;
10095 }
10096
10097 static void
10098 table_build_free(struct rte_swx_pipeline *p)
10099 {
10100         uint32_t i;
10101
10102         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10103                 struct thread *t = &p->threads[i];
10104                 uint32_t j;
10105
10106                 if (!t->tables)
10107                         continue;
10108
10109                 for (j = 0; j < p->n_tables; j++) {
10110                         struct table_runtime *r = &t->tables[j];
10111
10112                         free(r->mailbox);
10113                 }
10114
10115                 free(t->tables);
10116                 t->tables = NULL;
10117         }
10118
10119         if (p->table_stats) {
10120                 for (i = 0; i < p->n_tables; i++)
10121                         free(p->table_stats[i].n_pkts_action);
10122
10123                 free(p->table_stats);
10124         }
10125 }
10126
10127 static void
10128 table_free(struct rte_swx_pipeline *p)
10129 {
10130         table_build_free(p);
10131
10132         /* Tables. */
10133         for ( ; ; ) {
10134                 struct table *elem;
10135
10136                 elem = TAILQ_FIRST(&p->tables);
10137                 if (!elem)
10138                         break;
10139
10140                 TAILQ_REMOVE(&p->tables, elem, node);
10141                 free(elem->fields);
10142                 free(elem->actions);
10143                 free(elem->default_action_data);
10144                 free(elem);
10145         }
10146
10147         /* Table types. */
10148         for ( ; ; ) {
10149                 struct table_type *elem;
10150
10151                 elem = TAILQ_FIRST(&p->table_types);
10152                 if (!elem)
10153                         break;
10154
10155                 TAILQ_REMOVE(&p->table_types, elem, node);
10156                 free(elem);
10157         }
10158 }
10159
10160 /*
10161  * Selector.
10162  */
10163 static struct selector *
10164 selector_find(struct rte_swx_pipeline *p, const char *name)
10165 {
10166         struct selector *s;
10167
10168         TAILQ_FOREACH(s, &p->selectors, node)
10169                 if (strcmp(s->name, name) == 0)
10170                         return s;
10171
10172         return NULL;
10173 }
10174
10175 static struct selector *
10176 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
10177 {
10178         struct selector *s = NULL;
10179
10180         TAILQ_FOREACH(s, &p->selectors, node)
10181                 if (s->id == id)
10182                         return s;
10183
10184         return NULL;
10185 }
10186
10187 static int
10188 selector_fields_check(struct rte_swx_pipeline *p,
10189                       struct rte_swx_pipeline_selector_params *params,
10190                       struct header **header)
10191 {
10192         struct header *h0 = NULL;
10193         struct field *hf, *mf;
10194         uint32_t i;
10195
10196         /* Return if no selector fields. */
10197         if (!params->n_selector_fields || !params->selector_field_names)
10198                 return -EINVAL;
10199
10200         /* Check that all the selector fields either belong to the same header
10201          * or are all meta-data fields.
10202          */
10203         hf = header_field_parse(p, params->selector_field_names[0], &h0);
10204         mf = metadata_field_parse(p, params->selector_field_names[0]);
10205         if (!hf && !mf)
10206                 return -EINVAL;
10207
10208         for (i = 1; i < params->n_selector_fields; i++)
10209                 if (h0) {
10210                         struct header *h;
10211
10212                         hf = header_field_parse(p, params->selector_field_names[i], &h);
10213                         if (!hf || (h->id != h0->id))
10214                                 return -EINVAL;
10215                 } else {
10216                         mf = metadata_field_parse(p, params->selector_field_names[i]);
10217                         if (!mf)
10218                                 return -EINVAL;
10219                 }
10220
10221         /* Check that there are no duplicated match fields. */
10222         for (i = 0; i < params->n_selector_fields; i++) {
10223                 const char *field_name = params->selector_field_names[i];
10224                 uint32_t j;
10225
10226                 for (j = i + 1; j < params->n_selector_fields; j++)
10227                         if (!strcmp(params->selector_field_names[j], field_name))
10228                                 return -EINVAL;
10229         }
10230
10231         /* Return. */
10232         if (header)
10233                 *header = h0;
10234
10235         return 0;
10236 }
10237
10238 int
10239 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
10240                                  const char *name,
10241                                  struct rte_swx_pipeline_selector_params *params)
10242 {
10243         struct selector *s;
10244         struct header *selector_header = NULL;
10245         struct field *group_id_field, *member_id_field;
10246         uint32_t i;
10247         int status = 0;
10248
10249         CHECK(p, EINVAL);
10250
10251         CHECK_NAME(name, EINVAL);
10252         CHECK(!table_find(p, name), EEXIST);
10253         CHECK(!selector_find(p, name), EEXIST);
10254         CHECK(!learner_find(p, name), EEXIST);
10255
10256         CHECK(params, EINVAL);
10257
10258         CHECK_NAME(params->group_id_field_name, EINVAL);
10259         group_id_field = metadata_field_parse(p, params->group_id_field_name);
10260         CHECK(group_id_field, EINVAL);
10261
10262         for (i = 0; i < params->n_selector_fields; i++) {
10263                 const char *field_name = params->selector_field_names[i];
10264
10265                 CHECK_NAME(field_name, EINVAL);
10266         }
10267         status = selector_fields_check(p, params, &selector_header);
10268         if (status)
10269                 return status;
10270
10271         CHECK_NAME(params->member_id_field_name, EINVAL);
10272         member_id_field = metadata_field_parse(p, params->member_id_field_name);
10273         CHECK(member_id_field, EINVAL);
10274
10275         CHECK(params->n_groups_max, EINVAL);
10276
10277         CHECK(params->n_members_per_group_max, EINVAL);
10278
10279         /* Memory allocation. */
10280         s = calloc(1, sizeof(struct selector));
10281         if (!s) {
10282                 status = -ENOMEM;
10283                 goto error;
10284         }
10285
10286         s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
10287         if (!s->selector_fields) {
10288                 status = -ENOMEM;
10289                 goto error;
10290         }
10291
10292         /* Node initialization. */
10293         strcpy(s->name, name);
10294
10295         s->group_id_field = group_id_field;
10296
10297         for (i = 0; i < params->n_selector_fields; i++) {
10298                 const char *field_name = params->selector_field_names[i];
10299
10300                 s->selector_fields[i] = selector_header ?
10301                         header_field_parse(p, field_name, NULL) :
10302                         metadata_field_parse(p, field_name);
10303         }
10304
10305         s->n_selector_fields = params->n_selector_fields;
10306
10307         s->selector_header = selector_header;
10308
10309         s->member_id_field = member_id_field;
10310
10311         s->n_groups_max = params->n_groups_max;
10312
10313         s->n_members_per_group_max = params->n_members_per_group_max;
10314
10315         s->id = p->n_selectors;
10316
10317         /* Node add to tailq. */
10318         TAILQ_INSERT_TAIL(&p->selectors, s, node);
10319         p->n_selectors++;
10320
10321         return 0;
10322
10323 error:
10324         if (!s)
10325                 return status;
10326
10327         free(s->selector_fields);
10328
10329         free(s);
10330
10331         return status;
10332 }
10333
10334 static void
10335 selector_params_free(struct rte_swx_table_selector_params *params)
10336 {
10337         if (!params)
10338                 return;
10339
10340         free(params->selector_mask);
10341
10342         free(params);
10343 }
10344
10345 static struct rte_swx_table_selector_params *
10346 selector_table_params_get(struct selector *s)
10347 {
10348         struct rte_swx_table_selector_params *params = NULL;
10349         struct field *first, *last;
10350         uint32_t i;
10351
10352         /* Memory allocation. */
10353         params = calloc(1, sizeof(struct rte_swx_table_selector_params));
10354         if (!params)
10355                 goto error;
10356
10357         /* Group ID. */
10358         params->group_id_offset = s->group_id_field->offset / 8;
10359
10360         /* Find first (smallest offset) and last (biggest offset) selector fields. */
10361         first = s->selector_fields[0];
10362         last = s->selector_fields[0];
10363
10364         for (i = 0; i < s->n_selector_fields; i++) {
10365                 struct field *f = s->selector_fields[i];
10366
10367                 if (f->offset < first->offset)
10368                         first = f;
10369
10370                 if (f->offset > last->offset)
10371                         last = f;
10372         }
10373
10374         /* Selector offset and size. */
10375         params->selector_offset = first->offset / 8;
10376         params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
10377
10378         /* Memory allocation. */
10379         params->selector_mask = calloc(1, params->selector_size);
10380         if (!params->selector_mask)
10381                 goto error;
10382
10383         /* Selector mask. */
10384         for (i = 0; i < s->n_selector_fields; i++) {
10385                 struct field *f = s->selector_fields[i];
10386                 uint32_t start = (f->offset - first->offset) / 8;
10387                 size_t size = f->n_bits / 8;
10388
10389                 memset(&params->selector_mask[start], 0xFF, size);
10390         }
10391
10392         /* Member ID. */
10393         params->member_id_offset = s->member_id_field->offset / 8;
10394
10395         /* Maximum number of groups. */
10396         params->n_groups_max = s->n_groups_max;
10397
10398         /* Maximum number of members per group. */
10399         params->n_members_per_group_max = s->n_members_per_group_max;
10400
10401         return params;
10402
10403 error:
10404         selector_params_free(params);
10405         return NULL;
10406 }
10407
10408 static void
10409 selector_build_free(struct rte_swx_pipeline *p)
10410 {
10411         uint32_t i;
10412
10413         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10414                 struct thread *t = &p->threads[i];
10415                 uint32_t j;
10416
10417                 if (!t->selectors)
10418                         continue;
10419
10420                 for (j = 0; j < p->n_selectors; j++) {
10421                         struct selector_runtime *r = &t->selectors[j];
10422
10423                         free(r->mailbox);
10424                 }
10425
10426                 free(t->selectors);
10427                 t->selectors = NULL;
10428         }
10429
10430         free(p->selector_stats);
10431         p->selector_stats = NULL;
10432 }
10433
10434 static int
10435 selector_build(struct rte_swx_pipeline *p)
10436 {
10437         uint32_t i;
10438         int status = 0;
10439
10440         /* Per pipeline: selector statistics. */
10441         p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
10442         if (!p->selector_stats) {
10443                 status = -ENOMEM;
10444                 goto error;
10445         }
10446
10447         /* Per thread: selector run-time. */
10448         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10449                 struct thread *t = &p->threads[i];
10450                 struct selector *s;
10451
10452                 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
10453                 if (!t->selectors) {
10454                         status = -ENOMEM;
10455                         goto error;
10456                 }
10457
10458                 TAILQ_FOREACH(s, &p->selectors, node) {
10459                         struct selector_runtime *r = &t->selectors[s->id];
10460                         uint64_t size;
10461
10462                         /* r->mailbox. */
10463                         size = rte_swx_table_selector_mailbox_size_get();
10464                         if (size) {
10465                                 r->mailbox = calloc(1, size);
10466                                 if (!r->mailbox) {
10467                                         status = -ENOMEM;
10468                                         goto error;
10469                                 }
10470                         }
10471
10472                         /* r->group_id_buffer. */
10473                         r->group_id_buffer = &t->structs[p->metadata_struct_id];
10474
10475                         /* r->selector_buffer. */
10476                         r->selector_buffer = s->selector_header ?
10477                                 &t->structs[s->selector_header->struct_id] :
10478                                 &t->structs[p->metadata_struct_id];
10479
10480                         /* r->member_id_buffer. */
10481                         r->member_id_buffer = &t->structs[p->metadata_struct_id];
10482                 }
10483         }
10484
10485         return 0;
10486
10487 error:
10488         selector_build_free(p);
10489         return status;
10490 }
10491
10492 static void
10493 selector_free(struct rte_swx_pipeline *p)
10494 {
10495         selector_build_free(p);
10496
10497         /* Selector tables. */
10498         for ( ; ; ) {
10499                 struct selector *elem;
10500
10501                 elem = TAILQ_FIRST(&p->selectors);
10502                 if (!elem)
10503                         break;
10504
10505                 TAILQ_REMOVE(&p->selectors, elem, node);
10506                 free(elem->selector_fields);
10507                 free(elem);
10508         }
10509 }
10510
10511 /*
10512  * Learner table.
10513  */
10514 static struct learner *
10515 learner_find(struct rte_swx_pipeline *p, const char *name)
10516 {
10517         struct learner *l;
10518
10519         TAILQ_FOREACH(l, &p->learners, node)
10520                 if (!strcmp(l->name, name))
10521                         return l;
10522
10523         return NULL;
10524 }
10525
10526 static struct learner *
10527 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
10528 {
10529         struct learner *l = NULL;
10530
10531         TAILQ_FOREACH(l, &p->learners, node)
10532                 if (l->id == id)
10533                         return l;
10534
10535         return NULL;
10536 }
10537
10538 static int
10539 learner_match_fields_check(struct rte_swx_pipeline *p,
10540                            struct rte_swx_pipeline_learner_params *params,
10541                            struct header **header)
10542 {
10543         struct header *h0 = NULL;
10544         struct field *hf, *mf;
10545         uint32_t i;
10546
10547         /* Return if no match fields. */
10548         if (!params->n_fields || !params->field_names)
10549                 return -EINVAL;
10550
10551         /* Check that all the match fields either belong to the same header
10552          * or are all meta-data fields.
10553          */
10554         hf = header_field_parse(p, params->field_names[0], &h0);
10555         mf = metadata_field_parse(p, params->field_names[0]);
10556         if (!hf && !mf)
10557                 return -EINVAL;
10558
10559         for (i = 1; i < params->n_fields; i++)
10560                 if (h0) {
10561                         struct header *h;
10562
10563                         hf = header_field_parse(p, params->field_names[i], &h);
10564                         if (!hf || (h->id != h0->id))
10565                                 return -EINVAL;
10566                 } else {
10567                         mf = metadata_field_parse(p, params->field_names[i]);
10568                         if (!mf)
10569                                 return -EINVAL;
10570                 }
10571
10572         /* Check that there are no duplicated match fields. */
10573         for (i = 0; i < params->n_fields; i++) {
10574                 const char *field_name = params->field_names[i];
10575                 uint32_t j;
10576
10577                 for (j = i + 1; j < params->n_fields; j++)
10578                         if (!strcmp(params->field_names[j], field_name))
10579                                 return -EINVAL;
10580         }
10581
10582         /* Return. */
10583         if (header)
10584                 *header = h0;
10585
10586         return 0;
10587 }
10588
10589 static int
10590 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
10591 {
10592         struct struct_type *mst = p->metadata_st, *ast = a->st;
10593         struct field *mf, *af;
10594         uint32_t mf_pos, i;
10595
10596         if (!ast) {
10597                 if (mf_name)
10598                         return -EINVAL;
10599
10600                 return 0;
10601         }
10602
10603         /* Check that mf_name is the name of a valid meta-data field. */
10604         CHECK_NAME(mf_name, EINVAL);
10605         mf = metadata_field_parse(p, mf_name);
10606         CHECK(mf, EINVAL);
10607
10608         /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
10609          * all the action arguments.
10610          */
10611         mf_pos = mf - mst->fields;
10612         CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
10613
10614         /* Check that the size of each of the identified meta-data fields matches exactly the size
10615          * of the corresponding action argument.
10616          */
10617         for (i = 0; i < ast->n_fields; i++) {
10618                 mf = &mst->fields[mf_pos + i];
10619                 af = &ast->fields[i];
10620
10621                 CHECK(mf->n_bits == af->n_bits, EINVAL);
10622         }
10623
10624         return 0;
10625 }
10626
10627 static int
10628 learner_action_learning_check(struct rte_swx_pipeline *p,
10629                               struct action *action,
10630                               const char **action_names,
10631                               uint32_t n_actions)
10632 {
10633         uint32_t i;
10634
10635         /* For each "learn" instruction of the current action, check that the learned action (i.e.
10636          * the action passed as argument to the "learn" instruction) is also enabled for the
10637          * current learner table.
10638          */
10639         for (i = 0; i < action->n_instructions; i++) {
10640                 struct instruction *instr = &action->instructions[i];
10641                 uint32_t found = 0, j;
10642
10643                 if (instr->type != INSTR_LEARNER_LEARN)
10644                         continue;
10645
10646                 for (j = 0; j < n_actions; j++) {
10647                         struct action *a;
10648
10649                         a = action_find(p, action_names[j]);
10650                         if (!a)
10651                                 return -EINVAL;
10652
10653                         if (a->id == instr->learn.action_id)
10654                                 found = 1;
10655                 }
10656
10657                 if (!found)
10658                         return -EINVAL;
10659         }
10660
10661         return 0;
10662 }
10663
10664 int
10665 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
10666                               const char *name,
10667                               struct rte_swx_pipeline_learner_params *params,
10668                               uint32_t size,
10669                               uint32_t timeout)
10670 {
10671         struct learner *l = NULL;
10672         struct action *default_action;
10673         struct header *header = NULL;
10674         uint32_t action_data_size_max = 0, i;
10675         int status = 0;
10676
10677         CHECK(p, EINVAL);
10678
10679         CHECK_NAME(name, EINVAL);
10680         CHECK(!table_find(p, name), EEXIST);
10681         CHECK(!selector_find(p, name), EEXIST);
10682         CHECK(!learner_find(p, name), EEXIST);
10683
10684         CHECK(params, EINVAL);
10685
10686         /* Match checks. */
10687         status = learner_match_fields_check(p, params, &header);
10688         if (status)
10689                 return status;
10690
10691         /* Action checks. */
10692         CHECK(params->n_actions, EINVAL);
10693
10694         CHECK(params->action_names, EINVAL);
10695         for (i = 0; i < params->n_actions; i++) {
10696                 const char *action_name = params->action_names[i];
10697                 const char *action_field_name = params->action_field_names[i];
10698                 struct action *a;
10699                 uint32_t action_data_size;
10700
10701                 CHECK_NAME(action_name, EINVAL);
10702
10703                 a = action_find(p, action_name);
10704                 CHECK(a, EINVAL);
10705
10706                 status = learner_action_args_check(p, a, action_field_name);
10707                 if (status)
10708                         return status;
10709
10710                 status = learner_action_learning_check(p,
10711                                                        a,
10712                                                        params->action_names,
10713                                                        params->n_actions);
10714                 if (status)
10715                         return status;
10716
10717                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
10718                 if (action_data_size > action_data_size_max)
10719                         action_data_size_max = action_data_size;
10720         }
10721
10722         CHECK_NAME(params->default_action_name, EINVAL);
10723         for (i = 0; i < p->n_actions; i++)
10724                 if (!strcmp(params->action_names[i],
10725                             params->default_action_name))
10726                         break;
10727         CHECK(i < params->n_actions, EINVAL);
10728
10729         default_action = action_find(p, params->default_action_name);
10730         CHECK((default_action->st && params->default_action_data) ||
10731               !params->default_action_data, EINVAL);
10732
10733         /* Any other checks. */
10734         CHECK(size, EINVAL);
10735         CHECK(timeout, EINVAL);
10736
10737         /* Memory allocation. */
10738         l = calloc(1, sizeof(struct learner));
10739         if (!l)
10740                 goto nomem;
10741
10742         l->fields = calloc(params->n_fields, sizeof(struct field *));
10743         if (!l->fields)
10744                 goto nomem;
10745
10746         l->actions = calloc(params->n_actions, sizeof(struct action *));
10747         if (!l->actions)
10748                 goto nomem;
10749
10750         l->action_arg = calloc(params->n_actions, sizeof(struct field *));
10751         if (!l->action_arg)
10752                 goto nomem;
10753
10754         if (action_data_size_max) {
10755                 l->default_action_data = calloc(1, action_data_size_max);
10756                 if (!l->default_action_data)
10757                         goto nomem;
10758         }
10759
10760         /* Node initialization. */
10761         strcpy(l->name, name);
10762
10763         for (i = 0; i < params->n_fields; i++) {
10764                 const char *field_name = params->field_names[i];
10765
10766                 l->fields[i] = header ?
10767                         header_field_parse(p, field_name, NULL) :
10768                         metadata_field_parse(p, field_name);
10769         }
10770
10771         l->n_fields = params->n_fields;
10772
10773         l->header = header;
10774
10775         for (i = 0; i < params->n_actions; i++) {
10776                 const char *mf_name = params->action_field_names[i];
10777
10778                 l->actions[i] = action_find(p, params->action_names[i]);
10779
10780                 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
10781         }
10782
10783         l->default_action = default_action;
10784
10785         if (default_action->st)
10786                 memcpy(l->default_action_data,
10787                        params->default_action_data,
10788                        default_action->st->n_bits / 8);
10789
10790         l->n_actions = params->n_actions;
10791
10792         l->default_action_is_const = params->default_action_is_const;
10793
10794         l->action_data_size_max = action_data_size_max;
10795
10796         l->size = size;
10797
10798         l->timeout = timeout;
10799
10800         l->id = p->n_learners;
10801
10802         /* Node add to tailq. */
10803         TAILQ_INSERT_TAIL(&p->learners, l, node);
10804         p->n_learners++;
10805
10806         return 0;
10807
10808 nomem:
10809         if (!l)
10810                 return -ENOMEM;
10811
10812         free(l->action_arg);
10813         free(l->actions);
10814         free(l->fields);
10815         free(l);
10816
10817         return -ENOMEM;
10818 }
10819
10820 static void
10821 learner_params_free(struct rte_swx_table_learner_params *params)
10822 {
10823         if (!params)
10824                 return;
10825
10826         free(params->key_mask0);
10827
10828         free(params);
10829 }
10830
10831 static struct rte_swx_table_learner_params *
10832 learner_params_get(struct learner *l)
10833 {
10834         struct rte_swx_table_learner_params *params = NULL;
10835         struct field *first, *last;
10836         uint32_t i;
10837
10838         /* Memory allocation. */
10839         params = calloc(1, sizeof(struct rte_swx_table_learner_params));
10840         if (!params)
10841                 goto error;
10842
10843         /* Find first (smallest offset) and last (biggest offset) match fields. */
10844         first = l->fields[0];
10845         last = l->fields[0];
10846
10847         for (i = 0; i < l->n_fields; i++) {
10848                 struct field *f = l->fields[i];
10849
10850                 if (f->offset < first->offset)
10851                         first = f;
10852
10853                 if (f->offset > last->offset)
10854                         last = f;
10855         }
10856
10857         /* Key offset and size. */
10858         params->key_offset = first->offset / 8;
10859         params->key_size = (last->offset + last->n_bits - first->offset) / 8;
10860
10861         /* Memory allocation. */
10862         params->key_mask0 = calloc(1, params->key_size);
10863         if (!params->key_mask0)
10864                 goto error;
10865
10866         /* Key mask. */
10867         for (i = 0; i < l->n_fields; i++) {
10868                 struct field *f = l->fields[i];
10869                 uint32_t start = (f->offset - first->offset) / 8;
10870                 size_t size = f->n_bits / 8;
10871
10872                 memset(&params->key_mask0[start], 0xFF, size);
10873         }
10874
10875         /* Action data size. */
10876         params->action_data_size = l->action_data_size_max;
10877
10878         /* Maximum number of keys. */
10879         params->n_keys_max = l->size;
10880
10881         /* Timeout. */
10882         params->key_timeout = l->timeout;
10883
10884         return params;
10885
10886 error:
10887         learner_params_free(params);
10888         return NULL;
10889 }
10890
10891 static void
10892 learner_build_free(struct rte_swx_pipeline *p)
10893 {
10894         uint32_t i;
10895
10896         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10897                 struct thread *t = &p->threads[i];
10898                 uint32_t j;
10899
10900                 if (!t->learners)
10901                         continue;
10902
10903                 for (j = 0; j < p->n_learners; j++) {
10904                         struct learner_runtime *r = &t->learners[j];
10905
10906                         free(r->mailbox);
10907                         free(r->action_data);
10908                 }
10909
10910                 free(t->learners);
10911                 t->learners = NULL;
10912         }
10913
10914         if (p->learner_stats) {
10915                 for (i = 0; i < p->n_learners; i++)
10916                         free(p->learner_stats[i].n_pkts_action);
10917
10918                 free(p->learner_stats);
10919         }
10920 }
10921
10922 static int
10923 learner_build(struct rte_swx_pipeline *p)
10924 {
10925         uint32_t i;
10926         int status = 0;
10927
10928         /* Per pipeline: learner statistics. */
10929         p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
10930         CHECK(p->learner_stats, ENOMEM);
10931
10932         for (i = 0; i < p->n_learners; i++) {
10933                 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
10934                 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
10935         }
10936
10937         /* Per thread: learner run-time. */
10938         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10939                 struct thread *t = &p->threads[i];
10940                 struct learner *l;
10941
10942                 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
10943                 if (!t->learners) {
10944                         status = -ENOMEM;
10945                         goto error;
10946                 }
10947
10948                 TAILQ_FOREACH(l, &p->learners, node) {
10949                         struct learner_runtime *r = &t->learners[l->id];
10950                         uint64_t size;
10951                         uint32_t j;
10952
10953                         /* r->mailbox. */
10954                         size = rte_swx_table_learner_mailbox_size_get();
10955                         if (size) {
10956                                 r->mailbox = calloc(1, size);
10957                                 if (!r->mailbox) {
10958                                         status = -ENOMEM;
10959                                         goto error;
10960                                 }
10961                         }
10962
10963                         /* r->key. */
10964                         r->key = l->header ?
10965                                 &t->structs[l->header->struct_id] :
10966                                 &t->structs[p->metadata_struct_id];
10967
10968                         /* r->action_data. */
10969                         r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
10970                         if (!r->action_data) {
10971                                 status = -ENOMEM;
10972                                 goto error;
10973                         }
10974
10975                         for (j = 0; j < l->n_actions; j++) {
10976                                 struct action *a = l->actions[j];
10977                                 struct field *mf = l->action_arg[j];
10978                                 uint8_t *m = t->structs[p->metadata_struct_id];
10979
10980                                 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
10981                         }
10982                 }
10983         }
10984
10985         return 0;
10986
10987 error:
10988         learner_build_free(p);
10989         return status;
10990 }
10991
10992 static void
10993 learner_free(struct rte_swx_pipeline *p)
10994 {
10995         learner_build_free(p);
10996
10997         /* Learner tables. */
10998         for ( ; ; ) {
10999                 struct learner *l;
11000
11001                 l = TAILQ_FIRST(&p->learners);
11002                 if (!l)
11003                         break;
11004
11005                 TAILQ_REMOVE(&p->learners, l, node);
11006                 free(l->fields);
11007                 free(l->actions);
11008                 free(l->action_arg);
11009                 free(l->default_action_data);
11010                 free(l);
11011         }
11012 }
11013
11014 /*
11015  * Table state.
11016  */
11017 static int
11018 table_state_build(struct rte_swx_pipeline *p)
11019 {
11020         struct table *table;
11021         struct selector *s;
11022         struct learner *l;
11023
11024         p->table_state = calloc(p->n_tables + p->n_selectors,
11025                                 sizeof(struct rte_swx_table_state));
11026         CHECK(p->table_state, ENOMEM);
11027
11028         TAILQ_FOREACH(table, &p->tables, node) {
11029                 struct rte_swx_table_state *ts = &p->table_state[table->id];
11030
11031                 if (table->type) {
11032                         struct rte_swx_table_params *params;
11033
11034                         /* ts->obj. */
11035                         params = table_params_get(table);
11036                         CHECK(params, ENOMEM);
11037
11038                         ts->obj = table->type->ops.create(params,
11039                                 NULL,
11040                                 table->args,
11041                                 p->numa_node);
11042
11043                         table_params_free(params);
11044                         CHECK(ts->obj, ENODEV);
11045                 }
11046
11047                 /* ts->default_action_data. */
11048                 if (table->action_data_size_max) {
11049                         ts->default_action_data =
11050                                 malloc(table->action_data_size_max);
11051                         CHECK(ts->default_action_data, ENOMEM);
11052
11053                         memcpy(ts->default_action_data,
11054                                table->default_action_data,
11055                                table->action_data_size_max);
11056                 }
11057
11058                 /* ts->default_action_id. */
11059                 ts->default_action_id = table->default_action->id;
11060         }
11061
11062         TAILQ_FOREACH(s, &p->selectors, node) {
11063                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
11064                 struct rte_swx_table_selector_params *params;
11065
11066                 /* ts->obj. */
11067                 params = selector_table_params_get(s);
11068                 CHECK(params, ENOMEM);
11069
11070                 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
11071
11072                 selector_params_free(params);
11073                 CHECK(ts->obj, ENODEV);
11074         }
11075
11076         TAILQ_FOREACH(l, &p->learners, node) {
11077                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
11078                         p->n_selectors + l->id];
11079                 struct rte_swx_table_learner_params *params;
11080
11081                 /* ts->obj. */
11082                 params = learner_params_get(l);
11083                 CHECK(params, ENOMEM);
11084
11085                 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
11086                 learner_params_free(params);
11087                 CHECK(ts->obj, ENODEV);
11088
11089                 /* ts->default_action_data. */
11090                 if (l->action_data_size_max) {
11091                         ts->default_action_data = malloc(l->action_data_size_max);
11092                         CHECK(ts->default_action_data, ENOMEM);
11093
11094                         memcpy(ts->default_action_data,
11095                                l->default_action_data,
11096                                l->action_data_size_max);
11097                 }
11098
11099                 /* ts->default_action_id. */
11100                 ts->default_action_id = l->default_action->id;
11101         }
11102
11103         return 0;
11104 }
11105
11106 static void
11107 table_state_build_free(struct rte_swx_pipeline *p)
11108 {
11109         uint32_t i;
11110
11111         if (!p->table_state)
11112                 return;
11113
11114         for (i = 0; i < p->n_tables; i++) {
11115                 struct rte_swx_table_state *ts = &p->table_state[i];
11116                 struct table *table = table_find_by_id(p, i);
11117
11118                 /* ts->obj. */
11119                 if (table->type && ts->obj)
11120                         table->type->ops.free(ts->obj);
11121
11122                 /* ts->default_action_data. */
11123                 free(ts->default_action_data);
11124         }
11125
11126         for (i = 0; i < p->n_selectors; i++) {
11127                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
11128
11129                 /* ts->obj. */
11130                 if (ts->obj)
11131                         rte_swx_table_selector_free(ts->obj);
11132         }
11133
11134         for (i = 0; i < p->n_learners; i++) {
11135                 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
11136
11137                 /* ts->obj. */
11138                 if (ts->obj)
11139                         rte_swx_table_learner_free(ts->obj);
11140
11141                 /* ts->default_action_data. */
11142                 free(ts->default_action_data);
11143         }
11144
11145         free(p->table_state);
11146         p->table_state = NULL;
11147 }
11148
11149 static void
11150 table_state_free(struct rte_swx_pipeline *p)
11151 {
11152         table_state_build_free(p);
11153 }
11154
11155 /*
11156  * Register array.
11157  */
11158 static struct regarray *
11159 regarray_find(struct rte_swx_pipeline *p, const char *name)
11160 {
11161         struct regarray *elem;
11162
11163         TAILQ_FOREACH(elem, &p->regarrays, node)
11164                 if (!strcmp(elem->name, name))
11165                         return elem;
11166
11167         return NULL;
11168 }
11169
11170 static struct regarray *
11171 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
11172 {
11173         struct regarray *elem = NULL;
11174
11175         TAILQ_FOREACH(elem, &p->regarrays, node)
11176                 if (elem->id == id)
11177                         return elem;
11178
11179         return NULL;
11180 }
11181
11182 int
11183 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
11184                               const char *name,
11185                               uint32_t size,
11186                               uint64_t init_val)
11187 {
11188         struct regarray *r;
11189
11190         CHECK(p, EINVAL);
11191
11192         CHECK_NAME(name, EINVAL);
11193         CHECK(!regarray_find(p, name), EEXIST);
11194
11195         CHECK(size, EINVAL);
11196         size = rte_align32pow2(size);
11197
11198         /* Memory allocation. */
11199         r = calloc(1, sizeof(struct regarray));
11200         CHECK(r, ENOMEM);
11201
11202         /* Node initialization. */
11203         strcpy(r->name, name);
11204         r->init_val = init_val;
11205         r->size = size;
11206         r->id = p->n_regarrays;
11207
11208         /* Node add to tailq. */
11209         TAILQ_INSERT_TAIL(&p->regarrays, r, node);
11210         p->n_regarrays++;
11211
11212         return 0;
11213 }
11214
11215 static int
11216 regarray_build(struct rte_swx_pipeline *p)
11217 {
11218         struct regarray *regarray;
11219
11220         if (!p->n_regarrays)
11221                 return 0;
11222
11223         p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
11224         CHECK(p->regarray_runtime, ENOMEM);
11225
11226         TAILQ_FOREACH(regarray, &p->regarrays, node) {
11227                 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
11228                 uint32_t i;
11229
11230                 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
11231                                          RTE_CACHE_LINE_SIZE,
11232                                          p->numa_node);
11233                 CHECK(r->regarray, ENOMEM);
11234
11235                 if (regarray->init_val)
11236                         for (i = 0; i < regarray->size; i++)
11237                                 r->regarray[i] = regarray->init_val;
11238
11239                 r->size_mask = regarray->size - 1;
11240         }
11241
11242         return 0;
11243 }
11244
11245 static void
11246 regarray_build_free(struct rte_swx_pipeline *p)
11247 {
11248         uint32_t i;
11249
11250         if (!p->regarray_runtime)
11251                 return;
11252
11253         for (i = 0; i < p->n_regarrays; i++) {
11254                 struct regarray *regarray = regarray_find_by_id(p, i);
11255                 struct regarray_runtime *r = &p->regarray_runtime[i];
11256
11257                 env_free(r->regarray, regarray->size * sizeof(uint64_t));
11258         }
11259
11260         free(p->regarray_runtime);
11261         p->regarray_runtime = NULL;
11262 }
11263
11264 static void
11265 regarray_free(struct rte_swx_pipeline *p)
11266 {
11267         regarray_build_free(p);
11268
11269         for ( ; ; ) {
11270                 struct regarray *elem;
11271
11272                 elem = TAILQ_FIRST(&p->regarrays);
11273                 if (!elem)
11274                         break;
11275
11276                 TAILQ_REMOVE(&p->regarrays, elem, node);
11277                 free(elem);
11278         }
11279 }
11280
11281 /*
11282  * Meter array.
11283  */
11284 static struct meter_profile *
11285 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
11286 {
11287         struct meter_profile *elem;
11288
11289         TAILQ_FOREACH(elem, &p->meter_profiles, node)
11290                 if (!strcmp(elem->name, name))
11291                         return elem;
11292
11293         return NULL;
11294 }
11295
11296 static struct metarray *
11297 metarray_find(struct rte_swx_pipeline *p, const char *name)
11298 {
11299         struct metarray *elem;
11300
11301         TAILQ_FOREACH(elem, &p->metarrays, node)
11302                 if (!strcmp(elem->name, name))
11303                         return elem;
11304
11305         return NULL;
11306 }
11307
11308 static struct metarray *
11309 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
11310 {
11311         struct metarray *elem = NULL;
11312
11313         TAILQ_FOREACH(elem, &p->metarrays, node)
11314                 if (elem->id == id)
11315                         return elem;
11316
11317         return NULL;
11318 }
11319
11320 int
11321 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
11322                                  const char *name,
11323                                  uint32_t size)
11324 {
11325         struct metarray *m;
11326
11327         CHECK(p, EINVAL);
11328
11329         CHECK_NAME(name, EINVAL);
11330         CHECK(!metarray_find(p, name), EEXIST);
11331
11332         CHECK(size, EINVAL);
11333         size = rte_align32pow2(size);
11334
11335         /* Memory allocation. */
11336         m = calloc(1, sizeof(struct metarray));
11337         CHECK(m, ENOMEM);
11338
11339         /* Node initialization. */
11340         strcpy(m->name, name);
11341         m->size = size;
11342         m->id = p->n_metarrays;
11343
11344         /* Node add to tailq. */
11345         TAILQ_INSERT_TAIL(&p->metarrays, m, node);
11346         p->n_metarrays++;
11347
11348         return 0;
11349 }
11350
11351 struct meter_profile meter_profile_default = {
11352         .node = {0},
11353         .name = "",
11354         .params = {0},
11355
11356         .profile = {
11357                 .cbs = 10000,
11358                 .pbs = 10000,
11359                 .cir_period = 1,
11360                 .cir_bytes_per_period = 1,
11361                 .pir_period = 1,
11362                 .pir_bytes_per_period = 1,
11363         },
11364
11365         .n_users = 0,
11366 };
11367
11368 static void
11369 meter_init(struct meter *m)
11370 {
11371         memset(m, 0, sizeof(struct meter));
11372         rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
11373         m->profile = &meter_profile_default;
11374         m->color_mask = RTE_COLOR_GREEN;
11375
11376         meter_profile_default.n_users++;
11377 }
11378
11379 static int
11380 metarray_build(struct rte_swx_pipeline *p)
11381 {
11382         struct metarray *m;
11383
11384         if (!p->n_metarrays)
11385                 return 0;
11386
11387         p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
11388         CHECK(p->metarray_runtime, ENOMEM);
11389
11390         TAILQ_FOREACH(m, &p->metarrays, node) {
11391                 struct metarray_runtime *r = &p->metarray_runtime[m->id];
11392                 uint32_t i;
11393
11394                 r->metarray = env_malloc(m->size * sizeof(struct meter),
11395                                          RTE_CACHE_LINE_SIZE,
11396                                          p->numa_node);
11397                 CHECK(r->metarray, ENOMEM);
11398
11399                 for (i = 0; i < m->size; i++)
11400                         meter_init(&r->metarray[i]);
11401
11402                 r->size_mask = m->size - 1;
11403         }
11404
11405         return 0;
11406 }
11407
11408 static void
11409 metarray_build_free(struct rte_swx_pipeline *p)
11410 {
11411         uint32_t i;
11412
11413         if (!p->metarray_runtime)
11414                 return;
11415
11416         for (i = 0; i < p->n_metarrays; i++) {
11417                 struct metarray *m = metarray_find_by_id(p, i);
11418                 struct metarray_runtime *r = &p->metarray_runtime[i];
11419
11420                 env_free(r->metarray, m->size * sizeof(struct meter));
11421         }
11422
11423         free(p->metarray_runtime);
11424         p->metarray_runtime = NULL;
11425 }
11426
11427 static void
11428 metarray_free(struct rte_swx_pipeline *p)
11429 {
11430         metarray_build_free(p);
11431
11432         /* Meter arrays. */
11433         for ( ; ; ) {
11434                 struct metarray *elem;
11435
11436                 elem = TAILQ_FIRST(&p->metarrays);
11437                 if (!elem)
11438                         break;
11439
11440                 TAILQ_REMOVE(&p->metarrays, elem, node);
11441                 free(elem);
11442         }
11443
11444         /* Meter profiles. */
11445         for ( ; ; ) {
11446                 struct meter_profile *elem;
11447
11448                 elem = TAILQ_FIRST(&p->meter_profiles);
11449                 if (!elem)
11450                         break;
11451
11452                 TAILQ_REMOVE(&p->meter_profiles, elem, node);
11453                 free(elem);
11454         }
11455 }
11456
11457 /*
11458  * Pipeline.
11459  */
11460 int
11461 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
11462 {
11463         struct rte_swx_pipeline *pipeline;
11464
11465         /* Check input parameters. */
11466         CHECK(p, EINVAL);
11467
11468         /* Memory allocation. */
11469         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
11470         CHECK(pipeline, ENOMEM);
11471
11472         /* Initialization. */
11473         TAILQ_INIT(&pipeline->struct_types);
11474         TAILQ_INIT(&pipeline->port_in_types);
11475         TAILQ_INIT(&pipeline->ports_in);
11476         TAILQ_INIT(&pipeline->port_out_types);
11477         TAILQ_INIT(&pipeline->ports_out);
11478         TAILQ_INIT(&pipeline->extern_types);
11479         TAILQ_INIT(&pipeline->extern_objs);
11480         TAILQ_INIT(&pipeline->extern_funcs);
11481         TAILQ_INIT(&pipeline->headers);
11482         TAILQ_INIT(&pipeline->actions);
11483         TAILQ_INIT(&pipeline->table_types);
11484         TAILQ_INIT(&pipeline->tables);
11485         TAILQ_INIT(&pipeline->selectors);
11486         TAILQ_INIT(&pipeline->learners);
11487         TAILQ_INIT(&pipeline->regarrays);
11488         TAILQ_INIT(&pipeline->meter_profiles);
11489         TAILQ_INIT(&pipeline->metarrays);
11490
11491         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
11492         pipeline->numa_node = numa_node;
11493
11494         *p = pipeline;
11495         return 0;
11496 }
11497
11498 void
11499 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
11500 {
11501         if (!p)
11502                 return;
11503
11504         free(p->instructions);
11505
11506         metarray_free(p);
11507         regarray_free(p);
11508         table_state_free(p);
11509         learner_free(p);
11510         selector_free(p);
11511         table_free(p);
11512         action_free(p);
11513         metadata_free(p);
11514         header_free(p);
11515         extern_func_free(p);
11516         extern_obj_free(p);
11517         port_out_free(p);
11518         port_in_free(p);
11519         struct_free(p);
11520
11521         free(p);
11522 }
11523
11524 int
11525 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
11526                                      const char **instructions,
11527                                      uint32_t n_instructions)
11528 {
11529         int err;
11530         uint32_t i;
11531
11532         err = instruction_config(p, NULL, instructions, n_instructions);
11533         if (err)
11534                 return err;
11535
11536         /* Thread instruction pointer reset. */
11537         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
11538                 struct thread *t = &p->threads[i];
11539
11540                 thread_ip_reset(p, t);
11541         }
11542
11543         return 0;
11544 }
11545
11546 int
11547 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
11548 {
11549         int status;
11550
11551         CHECK(p, EINVAL);
11552         CHECK(p->build_done == 0, EEXIST);
11553
11554         status = port_in_build(p);
11555         if (status)
11556                 goto error;
11557
11558         status = port_out_build(p);
11559         if (status)
11560                 goto error;
11561
11562         status = struct_build(p);
11563         if (status)
11564                 goto error;
11565
11566         status = extern_obj_build(p);
11567         if (status)
11568                 goto error;
11569
11570         status = extern_func_build(p);
11571         if (status)
11572                 goto error;
11573
11574         status = header_build(p);
11575         if (status)
11576                 goto error;
11577
11578         status = metadata_build(p);
11579         if (status)
11580                 goto error;
11581
11582         status = action_build(p);
11583         if (status)
11584                 goto error;
11585
11586         status = table_build(p);
11587         if (status)
11588                 goto error;
11589
11590         status = selector_build(p);
11591         if (status)
11592                 goto error;
11593
11594         status = learner_build(p);
11595         if (status)
11596                 goto error;
11597
11598         status = table_state_build(p);
11599         if (status)
11600                 goto error;
11601
11602         status = regarray_build(p);
11603         if (status)
11604                 goto error;
11605
11606         status = metarray_build(p);
11607         if (status)
11608                 goto error;
11609
11610         p->build_done = 1;
11611         return 0;
11612
11613 error:
11614         metarray_build_free(p);
11615         regarray_build_free(p);
11616         table_state_build_free(p);
11617         learner_build_free(p);
11618         selector_build_free(p);
11619         table_build_free(p);
11620         action_build_free(p);
11621         metadata_build_free(p);
11622         header_build_free(p);
11623         extern_func_build_free(p);
11624         extern_obj_build_free(p);
11625         port_out_build_free(p);
11626         port_in_build_free(p);
11627         struct_build_free(p);
11628
11629         return status;
11630 }
11631
11632 void
11633 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
11634 {
11635         uint32_t i;
11636
11637         for (i = 0; i < n_instructions; i++)
11638                 instr_exec(p);
11639 }
11640
11641 void
11642 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
11643 {
11644         uint32_t i;
11645
11646         for (i = 0; i < p->n_ports_out; i++) {
11647                 struct port_out_runtime *port = &p->out[i];
11648
11649                 if (port->flush)
11650                         port->flush(port->obj);
11651         }
11652 }
11653
11654 /*
11655  * Control.
11656  */
11657 int
11658 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
11659                               struct rte_swx_ctl_pipeline_info *pipeline)
11660 {
11661         struct action *action;
11662         struct table *table;
11663         uint32_t n_actions = 0, n_tables = 0;
11664
11665         if (!p || !pipeline)
11666                 return -EINVAL;
11667
11668         TAILQ_FOREACH(action, &p->actions, node)
11669                 n_actions++;
11670
11671         TAILQ_FOREACH(table, &p->tables, node)
11672                 n_tables++;
11673
11674         pipeline->n_ports_in = p->n_ports_in;
11675         pipeline->n_ports_out = p->n_ports_out;
11676         pipeline->n_actions = n_actions;
11677         pipeline->n_tables = n_tables;
11678         pipeline->n_selectors = p->n_selectors;
11679         pipeline->n_learners = p->n_learners;
11680         pipeline->n_regarrays = p->n_regarrays;
11681         pipeline->n_metarrays = p->n_metarrays;
11682
11683         return 0;
11684 }
11685
11686 int
11687 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
11688 {
11689         if (!p || !numa_node)
11690                 return -EINVAL;
11691
11692         *numa_node = p->numa_node;
11693         return 0;
11694 }
11695
11696 int
11697 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
11698                             uint32_t action_id,
11699                             struct rte_swx_ctl_action_info *action)
11700 {
11701         struct action *a = NULL;
11702
11703         if (!p || (action_id >= p->n_actions) || !action)
11704                 return -EINVAL;
11705
11706         a = action_find_by_id(p, action_id);
11707         if (!a)
11708                 return -EINVAL;
11709
11710         strcpy(action->name, a->name);
11711         action->n_args = a->st ? a->st->n_fields : 0;
11712         return 0;
11713 }
11714
11715 int
11716 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
11717                                 uint32_t action_id,
11718                                 uint32_t action_arg_id,
11719                                 struct rte_swx_ctl_action_arg_info *action_arg)
11720 {
11721         struct action *a = NULL;
11722         struct field *arg = NULL;
11723
11724         if (!p || (action_id >= p->n_actions) || !action_arg)
11725                 return -EINVAL;
11726
11727         a = action_find_by_id(p, action_id);
11728         if (!a || !a->st || (action_arg_id >= a->st->n_fields))
11729                 return -EINVAL;
11730
11731         arg = &a->st->fields[action_arg_id];
11732         strcpy(action_arg->name, arg->name);
11733         action_arg->n_bits = arg->n_bits;
11734         action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
11735
11736         return 0;
11737 }
11738
11739 int
11740 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
11741                            uint32_t table_id,
11742                            struct rte_swx_ctl_table_info *table)
11743 {
11744         struct table *t = NULL;
11745
11746         if (!p || !table)
11747                 return -EINVAL;
11748
11749         t = table_find_by_id(p, table_id);
11750         if (!t)
11751                 return -EINVAL;
11752
11753         strcpy(table->name, t->name);
11754         strcpy(table->args, t->args);
11755         table->n_match_fields = t->n_fields;
11756         table->n_actions = t->n_actions;
11757         table->default_action_is_const = t->default_action_is_const;
11758         table->size = t->size;
11759         return 0;
11760 }
11761
11762 int
11763 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
11764         uint32_t table_id,
11765         uint32_t match_field_id,
11766         struct rte_swx_ctl_table_match_field_info *match_field)
11767 {
11768         struct table *t;
11769         struct match_field *f;
11770
11771         if (!p || (table_id >= p->n_tables) || !match_field)
11772                 return -EINVAL;
11773
11774         t = table_find_by_id(p, table_id);
11775         if (!t || (match_field_id >= t->n_fields))
11776                 return -EINVAL;
11777
11778         f = &t->fields[match_field_id];
11779         match_field->match_type = f->match_type;
11780         match_field->is_header = t->header ? 1 : 0;
11781         match_field->n_bits = f->field->n_bits;
11782         match_field->offset = f->field->offset;
11783
11784         return 0;
11785 }
11786
11787 int
11788 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
11789         uint32_t table_id,
11790         uint32_t table_action_id,
11791         struct rte_swx_ctl_table_action_info *table_action)
11792 {
11793         struct table *t;
11794
11795         if (!p || (table_id >= p->n_tables) || !table_action)
11796                 return -EINVAL;
11797
11798         t = table_find_by_id(p, table_id);
11799         if (!t || (table_action_id >= t->n_actions))
11800                 return -EINVAL;
11801
11802         table_action->action_id = t->actions[table_action_id]->id;
11803
11804         return 0;
11805 }
11806
11807 int
11808 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
11809                           uint32_t table_id,
11810                           struct rte_swx_table_ops *table_ops,
11811                           int *is_stub)
11812 {
11813         struct table *t;
11814
11815         if (!p || (table_id >= p->n_tables))
11816                 return -EINVAL;
11817
11818         t = table_find_by_id(p, table_id);
11819         if (!t)
11820                 return -EINVAL;
11821
11822         if (t->type) {
11823                 if (table_ops)
11824                         memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
11825                 *is_stub = 0;
11826         } else {
11827                 *is_stub = 1;
11828         }
11829
11830         return 0;
11831 }
11832
11833 int
11834 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
11835                               uint32_t selector_id,
11836                               struct rte_swx_ctl_selector_info *selector)
11837 {
11838         struct selector *s = NULL;
11839
11840         if (!p || !selector)
11841                 return -EINVAL;
11842
11843         s = selector_find_by_id(p, selector_id);
11844         if (!s)
11845                 return -EINVAL;
11846
11847         strcpy(selector->name, s->name);
11848
11849         selector->n_selector_fields = s->n_selector_fields;
11850         selector->n_groups_max = s->n_groups_max;
11851         selector->n_members_per_group_max = s->n_members_per_group_max;
11852
11853         return 0;
11854 }
11855
11856 int
11857 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
11858          uint32_t selector_id,
11859          struct rte_swx_ctl_table_match_field_info *field)
11860 {
11861         struct selector *s;
11862
11863         if (!p || (selector_id >= p->n_selectors) || !field)
11864                 return -EINVAL;
11865
11866         s = selector_find_by_id(p, selector_id);
11867         if (!s)
11868                 return -EINVAL;
11869
11870         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
11871         field->is_header = 0;
11872         field->n_bits = s->group_id_field->n_bits;
11873         field->offset = s->group_id_field->offset;
11874
11875         return 0;
11876 }
11877
11878 int
11879 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
11880          uint32_t selector_id,
11881          uint32_t selector_field_id,
11882          struct rte_swx_ctl_table_match_field_info *field)
11883 {
11884         struct selector *s;
11885         struct field *f;
11886
11887         if (!p || (selector_id >= p->n_selectors) || !field)
11888                 return -EINVAL;
11889
11890         s = selector_find_by_id(p, selector_id);
11891         if (!s || (selector_field_id >= s->n_selector_fields))
11892                 return -EINVAL;
11893
11894         f = s->selector_fields[selector_field_id];
11895         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
11896         field->is_header = s->selector_header ? 1 : 0;
11897         field->n_bits = f->n_bits;
11898         field->offset = f->offset;
11899
11900         return 0;
11901 }
11902
11903 int
11904 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
11905          uint32_t selector_id,
11906          struct rte_swx_ctl_table_match_field_info *field)
11907 {
11908         struct selector *s;
11909
11910         if (!p || (selector_id >= p->n_selectors) || !field)
11911                 return -EINVAL;
11912
11913         s = selector_find_by_id(p, selector_id);
11914         if (!s)
11915                 return -EINVAL;
11916
11917         field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
11918         field->is_header = 0;
11919         field->n_bits = s->member_id_field->n_bits;
11920         field->offset = s->member_id_field->offset;
11921
11922         return 0;
11923 }
11924
11925 int
11926 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
11927                              uint32_t learner_id,
11928                              struct rte_swx_ctl_learner_info *learner)
11929 {
11930         struct learner *l = NULL;
11931
11932         if (!p || !learner)
11933                 return -EINVAL;
11934
11935         l = learner_find_by_id(p, learner_id);
11936         if (!l)
11937                 return -EINVAL;
11938
11939         strcpy(learner->name, l->name);
11940
11941         learner->n_match_fields = l->n_fields;
11942         learner->n_actions = l->n_actions;
11943         learner->default_action_is_const = l->default_action_is_const;
11944         learner->size = l->size;
11945
11946         return 0;
11947 }
11948
11949 int
11950 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
11951                                          uint32_t learner_id,
11952                                          uint32_t match_field_id,
11953                                          struct rte_swx_ctl_table_match_field_info *match_field)
11954 {
11955         struct learner *l;
11956         struct field *f;
11957
11958         if (!p || (learner_id >= p->n_learners) || !match_field)
11959                 return -EINVAL;
11960
11961         l = learner_find_by_id(p, learner_id);
11962         if (!l || (match_field_id >= l->n_fields))
11963                 return -EINVAL;
11964
11965         f = l->fields[match_field_id];
11966         match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
11967         match_field->is_header = l->header ? 1 : 0;
11968         match_field->n_bits = f->n_bits;
11969         match_field->offset = f->offset;
11970
11971         return 0;
11972 }
11973
11974 int
11975 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
11976                                     uint32_t learner_id,
11977                                     uint32_t learner_action_id,
11978                                     struct rte_swx_ctl_table_action_info *learner_action)
11979 {
11980         struct learner *l;
11981
11982         if (!p || (learner_id >= p->n_learners) || !learner_action)
11983                 return -EINVAL;
11984
11985         l = learner_find_by_id(p, learner_id);
11986         if (!l || (learner_action_id >= l->n_actions))
11987                 return -EINVAL;
11988
11989         learner_action->action_id = l->actions[learner_action_id]->id;
11990
11991         return 0;
11992 }
11993
11994 int
11995 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
11996                                  struct rte_swx_table_state **table_state)
11997 {
11998         if (!p || !table_state || !p->build_done)
11999                 return -EINVAL;
12000
12001         *table_state = p->table_state;
12002         return 0;
12003 }
12004
12005 int
12006 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
12007                                  struct rte_swx_table_state *table_state)
12008 {
12009         if (!p || !table_state || !p->build_done)
12010                 return -EINVAL;
12011
12012         p->table_state = table_state;
12013         return 0;
12014 }
12015
12016 int
12017 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
12018                                         uint32_t port_id,
12019                                         struct rte_swx_port_in_stats *stats)
12020 {
12021         struct port_in *port;
12022
12023         if (!p || !stats)
12024                 return -EINVAL;
12025
12026         port = port_in_find(p, port_id);
12027         if (!port)
12028                 return -EINVAL;
12029
12030         port->type->ops.stats_read(port->obj, stats);
12031         return 0;
12032 }
12033
12034 int
12035 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
12036                                          uint32_t port_id,
12037                                          struct rte_swx_port_out_stats *stats)
12038 {
12039         struct port_out *port;
12040
12041         if (!p || !stats)
12042                 return -EINVAL;
12043
12044         port = port_out_find(p, port_id);
12045         if (!port)
12046                 return -EINVAL;
12047
12048         port->type->ops.stats_read(port->obj, stats);
12049         return 0;
12050 }
12051
12052 int
12053 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
12054                                       const char *table_name,
12055                                       struct rte_swx_table_stats *stats)
12056 {
12057         struct table *table;
12058         struct table_statistics *table_stats;
12059
12060         if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
12061                 return -EINVAL;
12062
12063         table = table_find(p, table_name);
12064         if (!table)
12065                 return -EINVAL;
12066
12067         table_stats = &p->table_stats[table->id];
12068
12069         memcpy(stats->n_pkts_action,
12070                table_stats->n_pkts_action,
12071                p->n_actions * sizeof(uint64_t));
12072
12073         stats->n_pkts_hit = table_stats->n_pkts_hit[1];
12074         stats->n_pkts_miss = table_stats->n_pkts_hit[0];
12075
12076         return 0;
12077 }
12078
12079 int
12080 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
12081         const char *selector_name,
12082         struct rte_swx_pipeline_selector_stats *stats)
12083 {
12084         struct selector *s;
12085
12086         if (!p || !selector_name || !selector_name[0] || !stats)
12087                 return -EINVAL;
12088
12089         s = selector_find(p, selector_name);
12090         if (!s)
12091                 return -EINVAL;
12092
12093         stats->n_pkts = p->selector_stats[s->id].n_pkts;
12094
12095         return 0;
12096 }
12097
12098 int
12099 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
12100                                         const char *learner_name,
12101                                         struct rte_swx_learner_stats *stats)
12102 {
12103         struct learner *l;
12104         struct learner_statistics *learner_stats;
12105
12106         if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
12107                 return -EINVAL;
12108
12109         l = learner_find(p, learner_name);
12110         if (!l)
12111                 return -EINVAL;
12112
12113         learner_stats = &p->learner_stats[l->id];
12114
12115         memcpy(stats->n_pkts_action,
12116                learner_stats->n_pkts_action,
12117                p->n_actions * sizeof(uint64_t));
12118
12119         stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
12120         stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
12121
12122         stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
12123         stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
12124
12125         stats->n_pkts_forget = learner_stats->n_pkts_forget;
12126
12127         return 0;
12128 }
12129
12130 int
12131 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
12132                               uint32_t regarray_id,
12133                               struct rte_swx_ctl_regarray_info *regarray)
12134 {
12135         struct regarray *r;
12136
12137         if (!p || !regarray)
12138                 return -EINVAL;
12139
12140         r = regarray_find_by_id(p, regarray_id);
12141         if (!r)
12142                 return -EINVAL;
12143
12144         strcpy(regarray->name, r->name);
12145         regarray->size = r->size;
12146         return 0;
12147 }
12148
12149 int
12150 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
12151                                    const char *regarray_name,
12152                                    uint32_t regarray_index,
12153                                    uint64_t *value)
12154 {
12155         struct regarray *regarray;
12156         struct regarray_runtime *r;
12157
12158         if (!p || !regarray_name || !value)
12159                 return -EINVAL;
12160
12161         regarray = regarray_find(p, regarray_name);
12162         if (!regarray || (regarray_index >= regarray->size))
12163                 return -EINVAL;
12164
12165         r = &p->regarray_runtime[regarray->id];
12166         *value = r->regarray[regarray_index];
12167         return 0;
12168 }
12169
12170 int
12171 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
12172                                    const char *regarray_name,
12173                                    uint32_t regarray_index,
12174                                    uint64_t value)
12175 {
12176         struct regarray *regarray;
12177         struct regarray_runtime *r;
12178
12179         if (!p || !regarray_name)
12180                 return -EINVAL;
12181
12182         regarray = regarray_find(p, regarray_name);
12183         if (!regarray || (regarray_index >= regarray->size))
12184                 return -EINVAL;
12185
12186         r = &p->regarray_runtime[regarray->id];
12187         r->regarray[regarray_index] = value;
12188         return 0;
12189 }
12190
12191 int
12192 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
12193                               uint32_t metarray_id,
12194                               struct rte_swx_ctl_metarray_info *metarray)
12195 {
12196         struct metarray *m;
12197
12198         if (!p || !metarray)
12199                 return -EINVAL;
12200
12201         m = metarray_find_by_id(p, metarray_id);
12202         if (!m)
12203                 return -EINVAL;
12204
12205         strcpy(metarray->name, m->name);
12206         metarray->size = m->size;
12207         return 0;
12208 }
12209
12210 int
12211 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
12212                               const char *name,
12213                               struct rte_meter_trtcm_params *params)
12214 {
12215         struct meter_profile *mp;
12216         int status;
12217
12218         CHECK(p, EINVAL);
12219         CHECK_NAME(name, EINVAL);
12220         CHECK(params, EINVAL);
12221         CHECK(!meter_profile_find(p, name), EEXIST);
12222
12223         /* Node allocation. */
12224         mp = calloc(1, sizeof(struct meter_profile));
12225         CHECK(mp, ENOMEM);
12226
12227         /* Node initialization. */
12228         strcpy(mp->name, name);
12229         memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
12230         status = rte_meter_trtcm_profile_config(&mp->profile, params);
12231         if (status) {
12232                 free(mp);
12233                 CHECK(0, EINVAL);
12234         }
12235
12236         /* Node add to tailq. */
12237         TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
12238
12239         return 0;
12240 }
12241
12242 int
12243 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
12244                                  const char *name)
12245 {
12246         struct meter_profile *mp;
12247
12248         CHECK(p, EINVAL);
12249         CHECK_NAME(name, EINVAL);
12250
12251         mp = meter_profile_find(p, name);
12252         CHECK(mp, EINVAL);
12253         CHECK(!mp->n_users, EBUSY);
12254
12255         /* Remove node from tailq. */
12256         TAILQ_REMOVE(&p->meter_profiles, mp, node);
12257         free(mp);
12258
12259         return 0;
12260 }
12261
12262 int
12263 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
12264                         const char *metarray_name,
12265                         uint32_t metarray_index)
12266 {
12267         struct meter_profile *mp_old;
12268         struct metarray *metarray;
12269         struct metarray_runtime *metarray_runtime;
12270         struct meter *m;
12271
12272         CHECK(p, EINVAL);
12273         CHECK_NAME(metarray_name, EINVAL);
12274
12275         metarray = metarray_find(p, metarray_name);
12276         CHECK(metarray, EINVAL);
12277         CHECK(metarray_index < metarray->size, EINVAL);
12278
12279         metarray_runtime = &p->metarray_runtime[metarray->id];
12280         m = &metarray_runtime->metarray[metarray_index];
12281         mp_old = m->profile;
12282
12283         meter_init(m);
12284
12285         mp_old->n_users--;
12286
12287         return 0;
12288 }
12289
12290 int
12291 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
12292                       const char *metarray_name,
12293                       uint32_t metarray_index,
12294                       const char *profile_name)
12295 {
12296         struct meter_profile *mp, *mp_old;
12297         struct metarray *metarray;
12298         struct metarray_runtime *metarray_runtime;
12299         struct meter *m;
12300
12301         CHECK(p, EINVAL);
12302         CHECK_NAME(metarray_name, EINVAL);
12303
12304         metarray = metarray_find(p, metarray_name);
12305         CHECK(metarray, EINVAL);
12306         CHECK(metarray_index < metarray->size, EINVAL);
12307
12308         mp = meter_profile_find(p, profile_name);
12309         CHECK(mp, EINVAL);
12310
12311         metarray_runtime = &p->metarray_runtime[metarray->id];
12312         m = &metarray_runtime->metarray[metarray_index];
12313         mp_old = m->profile;
12314
12315         memset(m, 0, sizeof(struct meter));
12316         rte_meter_trtcm_config(&m->m, &mp->profile);
12317         m->profile = mp;
12318         m->color_mask = RTE_COLORS;
12319
12320         mp->n_users++;
12321         mp_old->n_users--;
12322
12323         return 0;
12324 }
12325
12326 int
12327 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
12328                              const char *metarray_name,
12329                              uint32_t metarray_index,
12330                              struct rte_swx_ctl_meter_stats *stats)
12331 {
12332         struct metarray *metarray;
12333         struct metarray_runtime *metarray_runtime;
12334         struct meter *m;
12335
12336         CHECK(p, EINVAL);
12337         CHECK_NAME(metarray_name, EINVAL);
12338
12339         metarray = metarray_find(p, metarray_name);
12340         CHECK(metarray, EINVAL);
12341         CHECK(metarray_index < metarray->size, EINVAL);
12342
12343         CHECK(stats, EINVAL);
12344
12345         metarray_runtime = &p->metarray_runtime[metarray->id];
12346         m = &metarray_runtime->metarray[metarray_index];
12347
12348         memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
12349         memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
12350
12351         return 0;
12352 }