lib/pipeline/rte_swx_pipeline_internal.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2021 Intel Corporation
   3  */
   4 #ifndef __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
   5 #define __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
   6
   7 #include <inttypes.h>
   8 #include <string.h>
   9 #include <sys/queue.h>
  10
  11 #include <rte_byteorder.h>
  12 #include <rte_common.h>
  13 #include <rte_cycles.h>
  14 #include <rte_prefetch.h>
  15 #include <rte_meter.h>
  16
  17 #include <rte_swx_table_selector.h>
  18 #include <rte_swx_table_learner.h>
  19 #include <rte_swx_pipeline.h>
  20 #include <rte_swx_ctl.h>
  21
  22 #ifndef TRACE_LEVEL
  23 #define TRACE_LEVEL 0
  24 #endif
  25
  26 #if TRACE_LEVEL
  27 #define TRACE(...) printf(__VA_ARGS__)
  28 #else
  29 #define TRACE(...)
  30 #endif
  31
  32 /*
  33  * Environment.
  34  */
  35 #define ntoh64(x) rte_be_to_cpu_64(x)
  36 #define hton64(x) rte_cpu_to_be_64(x)
  37
  38 /*
  39  * Struct.
  40  */
  41 struct field {
  42         char name[RTE_SWX_NAME_SIZE];
  43         uint32_t n_bits;
  44         uint32_t offset;
  45         int var_size;
  46 };
  47
  48 struct struct_type {
  49         TAILQ_ENTRY(struct_type) node;
  50         char name[RTE_SWX_NAME_SIZE];
  51         struct field *fields;
  52         uint32_t n_fields;
  53         uint32_t n_bits;
  54         uint32_t n_bits_min;
  55         int var_size;
  56 };
  57
  58 TAILQ_HEAD(struct_type_tailq, struct_type);
  59
  60 /*
  61  * Input port.
  62  */
  63 struct port_in_type {
  64         TAILQ_ENTRY(port_in_type) node;
  65         char name[RTE_SWX_NAME_SIZE];
  66         struct rte_swx_port_in_ops ops;
  67 };
  68
  69 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  70
  71 struct port_in {
  72         TAILQ_ENTRY(port_in) node;
  73         struct port_in_type *type;
  74         void *obj;
  75         uint32_t id;
  76 };
  77
  78 TAILQ_HEAD(port_in_tailq, port_in);
  79
  80 struct port_in_runtime {
  81         rte_swx_port_in_pkt_rx_t pkt_rx;
  82         void *obj;
  83 };
  84
  85 /*
  86  * Output port.
  87  */
  88 struct port_out_type {
  89         TAILQ_ENTRY(port_out_type) node;
  90         char name[RTE_SWX_NAME_SIZE];
  91         struct rte_swx_port_out_ops ops;
  92 };
  93
  94 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  95
  96 struct port_out {
  97         TAILQ_ENTRY(port_out) node;
  98         struct port_out_type *type;
  99         void *obj;
 100         uint32_t id;
 101 };
 102
 103 TAILQ_HEAD(port_out_tailq, port_out);
 104
 105 struct port_out_runtime {
 106         rte_swx_port_out_pkt_tx_t pkt_tx;
 107         rte_swx_port_out_flush_t flush;
 108         void *obj;
 109 };
 110
 111 /*
 112  * Extern object.
 113  */
 114 struct extern_type_member_func {
 115         TAILQ_ENTRY(extern_type_member_func) node;
 116         char name[RTE_SWX_NAME_SIZE];
 117         rte_swx_extern_type_member_func_t func;
 118         uint32_t id;
 119 };
 120
 121 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 122
 123 struct extern_type {
 124         TAILQ_ENTRY(extern_type) node;
 125         char name[RTE_SWX_NAME_SIZE];
 126         struct struct_type *mailbox_struct_type;
 127         rte_swx_extern_type_constructor_t constructor;
 128         rte_swx_extern_type_destructor_t destructor;
 129         struct extern_type_member_func_tailq funcs;
 130         uint32_t n_funcs;
 131 };
 132
 133 TAILQ_HEAD(extern_type_tailq, extern_type);
 134
 135 struct extern_obj {
 136         TAILQ_ENTRY(extern_obj) node;
 137         char name[RTE_SWX_NAME_SIZE];
 138         struct extern_type *type;
 139         void *obj;
 140         uint32_t struct_id;
 141         uint32_t id;
 142 };
 143
 144 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 145
 146 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 147 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 148 #endif
 149
 150 struct extern_obj_runtime {
 151         void *obj;
 152         uint8_t *mailbox;
 153         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 154 };
 155
 156 /*
 157  * Extern function.
 158  */
 159 struct extern_func {
 160         TAILQ_ENTRY(extern_func) node;
 161         char name[RTE_SWX_NAME_SIZE];
 162         struct struct_type *mailbox_struct_type;
 163         rte_swx_extern_func_t func;
 164         uint32_t struct_id;
 165         uint32_t id;
 166 };
 167
 168 TAILQ_HEAD(extern_func_tailq, extern_func);
 169
 170 struct extern_func_runtime {
 171         uint8_t *mailbox;
 172         rte_swx_extern_func_t func;
 173 };
 174
 175 /*
 176  * Header.
 177  */
 178 struct header {
 179         TAILQ_ENTRY(header) node;
 180         char name[RTE_SWX_NAME_SIZE];
 181         struct struct_type *st;
 182         uint32_t struct_id;
 183         uint32_t id;
 184 };
 185
 186 TAILQ_HEAD(header_tailq, header);
 187
 188 struct header_runtime {
 189         uint8_t *ptr0;
 190         uint32_t n_bytes;
 191 };
 192
 193 struct header_out_runtime {
 194         uint8_t *ptr0;
 195         uint8_t *ptr;
 196         uint32_t n_bytes;
 197 };
 198
 199 /*
 200  * Instruction.
 201  */
 202
 203 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 204  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 205  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 206  * when transferred to packet meta-data and in NBO when transferred to packet
 207  * headers.
 208  */
 209
 210 /* Notation conventions:
 211  *    -Header field: H = h.header.field (dst/src)
 212  *    -Meta-data field: M = m.field (dst/src)
 213  *    -Extern object mailbox field: E = e.field (dst/src)
 214  *    -Extern function mailbox field: F = f.field (dst/src)
 215  *    -Table action data field: T = t.field (src only)
 216  *    -Immediate value: I = 32-bit unsigned value (src only)
 217  */
 218
 219 enum instruction_type {
 220         /* rx m.port_in */
 221         INSTR_RX,
 222
 223         /* tx port_out
 224          * port_out = MI
 225          */
 226         INSTR_TX,   /* port_out = M */
 227         INSTR_TX_I, /* port_out = I */
 228
 229         /* extract h.header */
 230         INSTR_HDR_EXTRACT,
 231         INSTR_HDR_EXTRACT2,
 232         INSTR_HDR_EXTRACT3,
 233         INSTR_HDR_EXTRACT4,
 234         INSTR_HDR_EXTRACT5,
 235         INSTR_HDR_EXTRACT6,
 236         INSTR_HDR_EXTRACT7,
 237         INSTR_HDR_EXTRACT8,
 238
 239         /* extract h.header m.last_field_size */
 240         INSTR_HDR_EXTRACT_M,
 241
 242         /* lookahead h.header */
 243         INSTR_HDR_LOOKAHEAD,
 244
 245         /* emit h.header */
 246         INSTR_HDR_EMIT,
 247         INSTR_HDR_EMIT_TX,
 248         INSTR_HDR_EMIT2_TX,
 249         INSTR_HDR_EMIT3_TX,
 250         INSTR_HDR_EMIT4_TX,
 251         INSTR_HDR_EMIT5_TX,
 252         INSTR_HDR_EMIT6_TX,
 253         INSTR_HDR_EMIT7_TX,
 254         INSTR_HDR_EMIT8_TX,
 255
 256         /* validate h.header */
 257         INSTR_HDR_VALIDATE,
 258
 259         /* invalidate h.header */
 260         INSTR_HDR_INVALIDATE,
 261
 262         /* mov dst src
 263          * dst = src
 264          * dst = HMEF, src = HMEFTI
 265          */
 266         INSTR_MOV,    /* dst = MEF, src = MEFT */
 267         INSTR_MOV_MH, /* dst = MEF, src = H */
 268         INSTR_MOV_HM, /* dst = H, src = MEFT */
 269         INSTR_MOV_HH, /* dst = H, src = H */
 270         INSTR_MOV_I,  /* dst = HMEF, src = I */
 271
 272         /* dma h.header t.field
 273          * memcpy(h.header, t.field, sizeof(h.header))
 274          */
 275         INSTR_DMA_HT,
 276         INSTR_DMA_HT2,
 277         INSTR_DMA_HT3,
 278         INSTR_DMA_HT4,
 279         INSTR_DMA_HT5,
 280         INSTR_DMA_HT6,
 281         INSTR_DMA_HT7,
 282         INSTR_DMA_HT8,
 283
 284         /* add dst src
 285          * dst += src
 286          * dst = HMEF, src = HMEFTI
 287          */
 288         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 289         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 290         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 291         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 292         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 293         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 294
 295         /* sub dst src
 296          * dst -= src
 297          * dst = HMEF, src = HMEFTI
 298          */
 299         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 300         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 301         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 302         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 303         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 304         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 305
 306         /* ckadd dst src
 307          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 308          * dst = H, src = {H, h.header}
 309          */
 310         INSTR_ALU_CKADD_FIELD,    /* src = H */
 311         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 312         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 313
 314         /* cksub dst src
 315          * dst = dst '- src
 316          * dst = H, src = H
 317          */
 318         INSTR_ALU_CKSUB_FIELD,
 319
 320         /* and dst src
 321          * dst &= src
 322          * dst = HMEF, src = HMEFTI
 323          */
 324         INSTR_ALU_AND,    /* dst = MEF, src = MEFT */
 325         INSTR_ALU_AND_MH, /* dst = MEF, src = H */
 326         INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
 327         INSTR_ALU_AND_HH, /* dst = H, src = H */
 328         INSTR_ALU_AND_I,  /* dst = HMEF, src = I */
 329
 330         /* or dst src
 331          * dst |= src
 332          * dst = HMEF, src = HMEFTI
 333          */
 334         INSTR_ALU_OR,    /* dst = MEF, src = MEFT */
 335         INSTR_ALU_OR_MH, /* dst = MEF, src = H */
 336         INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
 337         INSTR_ALU_OR_HH, /* dst = H, src = H */
 338         INSTR_ALU_OR_I,  /* dst = HMEF, src = I */
 339
 340         /* xor dst src
 341          * dst ^= src
 342          * dst = HMEF, src = HMEFTI
 343          */
 344         INSTR_ALU_XOR,    /* dst = MEF, src = MEFT */
 345         INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
 346         INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
 347         INSTR_ALU_XOR_HH, /* dst = H, src = H */
 348         INSTR_ALU_XOR_I,  /* dst = HMEF, src = I */
 349
 350         /* shl dst src
 351          * dst <<= src
 352          * dst = HMEF, src = HMEFTI
 353          */
 354         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 355         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 356         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 357         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 358         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 359         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 360
 361         /* shr dst src
 362          * dst >>= src
 363          * dst = HMEF, src = HMEFTI
 364          */
 365         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 366         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 367         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 368         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 369         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 370         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 371
 372         /* regprefetch REGARRAY index
 373          * prefetch REGARRAY[index]
 374          * index = HMEFTI
 375          */
 376         INSTR_REGPREFETCH_RH, /* index = H */
 377         INSTR_REGPREFETCH_RM, /* index = MEFT */
 378         INSTR_REGPREFETCH_RI, /* index = I */
 379
 380         /* regrd dst REGARRAY index
 381          * dst = REGARRAY[index]
 382          * dst = HMEF, index = HMEFTI
 383          */
 384         INSTR_REGRD_HRH, /* dst = H, index = H */
 385         INSTR_REGRD_HRM, /* dst = H, index = MEFT */
 386         INSTR_REGRD_HRI, /* dst = H, index = I */
 387         INSTR_REGRD_MRH, /* dst = MEF, index = H */
 388         INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
 389         INSTR_REGRD_MRI, /* dst = MEF, index = I */
 390
 391         /* regwr REGARRAY index src
 392          * REGARRAY[index] = src
 393          * index = HMEFTI, src = HMEFTI
 394          */
 395         INSTR_REGWR_RHH, /* index = H, src = H */
 396         INSTR_REGWR_RHM, /* index = H, src = MEFT */
 397         INSTR_REGWR_RHI, /* index = H, src = I */
 398         INSTR_REGWR_RMH, /* index = MEFT, src = H */
 399         INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
 400         INSTR_REGWR_RMI, /* index = MEFT, src = I */
 401         INSTR_REGWR_RIH, /* index = I, src = H */
 402         INSTR_REGWR_RIM, /* index = I, src = MEFT */
 403         INSTR_REGWR_RII, /* index = I, src = I */
 404
 405         /* regadd REGARRAY index src
 406          * REGARRAY[index] += src
 407          * index = HMEFTI, src = HMEFTI
 408          */
 409         INSTR_REGADD_RHH, /* index = H, src = H */
 410         INSTR_REGADD_RHM, /* index = H, src = MEFT */
 411         INSTR_REGADD_RHI, /* index = H, src = I */
 412         INSTR_REGADD_RMH, /* index = MEFT, src = H */
 413         INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
 414         INSTR_REGADD_RMI, /* index = MEFT, src = I */
 415         INSTR_REGADD_RIH, /* index = I, src = H */
 416         INSTR_REGADD_RIM, /* index = I, src = MEFT */
 417         INSTR_REGADD_RII, /* index = I, src = I */
 418
 419         /* metprefetch METARRAY index
 420          * prefetch METARRAY[index]
 421          * index = HMEFTI
 422          */
 423         INSTR_METPREFETCH_H, /* index = H */
 424         INSTR_METPREFETCH_M, /* index = MEFT */
 425         INSTR_METPREFETCH_I, /* index = I */
 426
 427         /* meter METARRAY index length color_in color_out
 428          * color_out = meter(METARRAY[index], length, color_in)
 429          * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
 430          */
 431         INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
 432         INSTR_METER_HHI, /* index = H, length = H, color_in = I */
 433         INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
 434         INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
 435         INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
 436         INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
 437         INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
 438         INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
 439         INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
 440         INSTR_METER_IHI, /* index = I, length = H, color_in = I */
 441         INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
 442         INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
 443
 444         /* table TABLE */
 445         INSTR_TABLE,
 446         INSTR_SELECTOR,
 447         INSTR_LEARNER,
 448
 449         /* learn LEARNER ACTION_NAME */
 450         INSTR_LEARNER_LEARN,
 451
 452         /* forget */
 453         INSTR_LEARNER_FORGET,
 454
 455         /* extern e.obj.func */
 456         INSTR_EXTERN_OBJ,
 457
 458         /* extern f.func */
 459         INSTR_EXTERN_FUNC,
 460
 461         /* jmp LABEL
 462          * Unconditional jump
 463          */
 464         INSTR_JMP,
 465
 466         /* jmpv LABEL h.header
 467          * Jump if header is valid
 468          */
 469         INSTR_JMP_VALID,
 470
 471         /* jmpnv LABEL h.header
 472          * Jump if header is invalid
 473          */
 474         INSTR_JMP_INVALID,
 475
 476         /* jmph LABEL
 477          * Jump if table lookup hit
 478          */
 479         INSTR_JMP_HIT,
 480
 481         /* jmpnh LABEL
 482          * Jump if table lookup miss
 483          */
 484         INSTR_JMP_MISS,
 485
 486         /* jmpa LABEL ACTION
 487          * Jump if action run
 488          */
 489         INSTR_JMP_ACTION_HIT,
 490
 491         /* jmpna LABEL ACTION
 492          * Jump if action not run
 493          */
 494         INSTR_JMP_ACTION_MISS,
 495
 496         /* jmpeq LABEL a b
 497          * Jump if a is equal to b
 498          * a = HMEFT, b = HMEFTI
 499          */
 500         INSTR_JMP_EQ,    /* a = MEFT, b = MEFT */
 501         INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
 502         INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
 503         INSTR_JMP_EQ_HH, /* a = H, b = H */
 504         INSTR_JMP_EQ_I,  /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 505
 506         /* jmpneq LABEL a b
 507          * Jump if a is not equal to b
 508          * a = HMEFT, b = HMEFTI
 509          */
 510         INSTR_JMP_NEQ,    /* a = MEFT, b = MEFT */
 511         INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
 512         INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
 513         INSTR_JMP_NEQ_HH, /* a = H, b = H */
 514         INSTR_JMP_NEQ_I,  /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
 515
 516         /* jmplt LABEL a b
 517          * Jump if a is less than b
 518          * a = HMEFT, b = HMEFTI
 519          */
 520         INSTR_JMP_LT,    /* a = MEFT, b = MEFT */
 521         INSTR_JMP_LT_MH, /* a = MEFT, b = H */
 522         INSTR_JMP_LT_HM, /* a = H, b = MEFT */
 523         INSTR_JMP_LT_HH, /* a = H, b = H */
 524         INSTR_JMP_LT_MI, /* a = MEFT, b = I */
 525         INSTR_JMP_LT_HI, /* a = H, b = I */
 526
 527         /* jmpgt LABEL a b
 528          * Jump if a is greater than b
 529          * a = HMEFT, b = HMEFTI
 530          */
 531         INSTR_JMP_GT,    /* a = MEFT, b = MEFT */
 532         INSTR_JMP_GT_MH, /* a = MEFT, b = H */
 533         INSTR_JMP_GT_HM, /* a = H, b = MEFT */
 534         INSTR_JMP_GT_HH, /* a = H, b = H */
 535         INSTR_JMP_GT_MI, /* a = MEFT, b = I */
 536         INSTR_JMP_GT_HI, /* a = H, b = I */
 537
 538         /* return
 539          * Return from action
 540          */
 541         INSTR_RETURN,
 542 };
 543
 544 struct instr_operand {
 545         uint8_t struct_id;
 546         uint8_t n_bits;
 547         uint8_t offset;
 548         uint8_t pad;
 549 };
 550
 551 struct instr_io {
 552         struct {
 553                 union {
 554                         struct {
 555                                 uint8_t offset;
 556                                 uint8_t n_bits;
 557                                 uint8_t pad[2];
 558                         };
 559
 560                         uint32_t val;
 561                 };
 562         } io;
 563
 564         struct {
 565                 uint8_t header_id[8];
 566                 uint8_t struct_id[8];
 567                 uint8_t n_bytes[8];
 568         } hdr;
 569 };
 570
 571 struct instr_hdr_validity {
 572         uint8_t header_id;
 573 };
 574
 575 struct instr_table {
 576         uint8_t table_id;
 577 };
 578
 579 struct instr_learn {
 580         uint8_t action_id;
 581 };
 582
 583 struct instr_extern_obj {
 584         uint8_t ext_obj_id;
 585         uint8_t func_id;
 586 };
 587
 588 struct instr_extern_func {
 589         uint8_t ext_func_id;
 590 };
 591
 592 struct instr_dst_src {
 593         struct instr_operand dst;
 594         union {
 595                 struct instr_operand src;
 596                 uint64_t src_val;
 597         };
 598 };
 599
 600 struct instr_regarray {
 601         uint8_t regarray_id;
 602         uint8_t pad[3];
 603
 604         union {
 605                 struct instr_operand idx;
 606                 uint32_t idx_val;
 607         };
 608
 609         union {
 610                 struct instr_operand dstsrc;
 611                 uint64_t dstsrc_val;
 612         };
 613 };
 614
 615 struct instr_meter {
 616         uint8_t metarray_id;
 617         uint8_t pad[3];
 618
 619         union {
 620                 struct instr_operand idx;
 621                 uint32_t idx_val;
 622         };
 623
 624         struct instr_operand length;
 625
 626         union {
 627                 struct instr_operand color_in;
 628                 uint32_t color_in_val;
 629         };
 630
 631         struct instr_operand color_out;
 632 };
 633
 634 struct instr_dma {
 635         struct {
 636                 uint8_t header_id[8];
 637                 uint8_t struct_id[8];
 638         } dst;
 639
 640         struct {
 641                 uint8_t offset[8];
 642         } src;
 643
 644         uint16_t n_bytes[8];
 645 };
 646
 647 struct instr_jmp {
 648         struct instruction *ip;
 649
 650         union {
 651                 struct instr_operand a;
 652                 uint8_t header_id;
 653                 uint8_t action_id;
 654         };
 655
 656         union {
 657                 struct instr_operand b;
 658                 uint64_t b_val;
 659         };
 660 };
 661
 662 struct instruction {
 663         enum instruction_type type;
 664         union {
 665                 struct instr_io io;
 666                 struct instr_hdr_validity valid;
 667                 struct instr_dst_src mov;
 668                 struct instr_regarray regarray;
 669                 struct instr_meter meter;
 670                 struct instr_dma dma;
 671                 struct instr_dst_src alu;
 672                 struct instr_table table;
 673                 struct instr_learn learn;
 674                 struct instr_extern_obj ext_obj;
 675                 struct instr_extern_func ext_func;
 676                 struct instr_jmp jmp;
 677         };
 678 };
 679
 680 struct instruction_data {
 681         char label[RTE_SWX_NAME_SIZE];
 682         char jmp_label[RTE_SWX_NAME_SIZE];
 683         uint32_t n_users; /* user = jmp instruction to this instruction. */
 684         int invalid;
 685 };
 686
 687 /*
 688  * Action.
 689  */
 690 struct action {
 691         TAILQ_ENTRY(action) node;
 692         char name[RTE_SWX_NAME_SIZE];
 693         struct struct_type *st;
 694         int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */
 695         struct instruction *instructions;
 696         uint32_t n_instructions;
 697         uint32_t id;
 698 };
 699
 700 TAILQ_HEAD(action_tailq, action);
 701
 702 /*
 703  * Table.
 704  */
 705 struct table_type {
 706         TAILQ_ENTRY(table_type) node;
 707         char name[RTE_SWX_NAME_SIZE];
 708         enum rte_swx_table_match_type match_type;
 709         struct rte_swx_table_ops ops;
 710 };
 711
 712 TAILQ_HEAD(table_type_tailq, table_type);
 713
 714 struct match_field {
 715         enum rte_swx_table_match_type match_type;
 716         struct field *field;
 717 };
 718
 719 struct table {
 720         TAILQ_ENTRY(table) node;
 721         char name[RTE_SWX_NAME_SIZE];
 722         char args[RTE_SWX_NAME_SIZE];
 723         struct table_type *type; /* NULL when n_fields == 0. */
 724
 725         /* Match. */
 726         struct match_field *fields;
 727         uint32_t n_fields;
 728         struct header *header; /* Only valid when n_fields > 0. */
 729
 730         /* Action. */
 731         struct action **actions;
 732         struct action *default_action;
 733         uint8_t *default_action_data;
 734         uint32_t n_actions;
 735         int default_action_is_const;
 736         uint32_t action_data_size_max;
 737
 738         uint32_t size;
 739         uint32_t id;
 740 };
 741
 742 TAILQ_HEAD(table_tailq, table);
 743
 744 struct table_runtime {
 745         rte_swx_table_lookup_t func;
 746         void *mailbox;
 747         uint8_t **key;
 748 };
 749
 750 struct table_statistics {
 751         uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
 752         uint64_t *n_pkts_action;
 753 };
 754
 755 /*
 756  * Selector.
 757  */
 758 struct selector {
 759         TAILQ_ENTRY(selector) node;
 760         char name[RTE_SWX_NAME_SIZE];
 761
 762         struct field *group_id_field;
 763         struct field **selector_fields;
 764         uint32_t n_selector_fields;
 765         struct header *selector_header;
 766         struct field *member_id_field;
 767
 768         uint32_t n_groups_max;
 769         uint32_t n_members_per_group_max;
 770
 771         uint32_t id;
 772 };
 773
 774 TAILQ_HEAD(selector_tailq, selector);
 775
 776 struct selector_runtime {
 777         void *mailbox;
 778         uint8_t **group_id_buffer;
 779         uint8_t **selector_buffer;
 780         uint8_t **member_id_buffer;
 781 };
 782
 783 struct selector_statistics {
 784         uint64_t n_pkts;
 785 };
 786
 787 /*
 788  * Learner table.
 789  */
 790 struct learner {
 791         TAILQ_ENTRY(learner) node;
 792         char name[RTE_SWX_NAME_SIZE];
 793
 794         /* Match. */
 795         struct field **fields;
 796         uint32_t n_fields;
 797         struct header *header;
 798
 799         /* Action. */
 800         struct action **actions;
 801         struct field **action_arg;
 802         struct action *default_action;
 803         uint8_t *default_action_data;
 804         uint32_t n_actions;
 805         int default_action_is_const;
 806         uint32_t action_data_size_max;
 807
 808         uint32_t size;
 809         uint32_t timeout;
 810         uint32_t id;
 811 };
 812
 813 TAILQ_HEAD(learner_tailq, learner);
 814
 815 struct learner_runtime {
 816         void *mailbox;
 817         uint8_t **key;
 818         uint8_t **action_data;
 819 };
 820
 821 struct learner_statistics {
 822         uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
 823         uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */
 824         uint64_t n_pkts_forget;
 825         uint64_t *n_pkts_action;
 826 };
 827
 828 /*
 829  * Register array.
 830  */
 831 struct regarray {
 832         TAILQ_ENTRY(regarray) node;
 833         char name[RTE_SWX_NAME_SIZE];
 834         uint64_t init_val;
 835         uint32_t size;
 836         uint32_t id;
 837 };
 838
 839 TAILQ_HEAD(regarray_tailq, regarray);
 840
 841 struct regarray_runtime {
 842         uint64_t *regarray;
 843         uint32_t size_mask;
 844 };
 845
 846 /*
 847  * Meter array.
 848  */
 849 struct meter_profile {
 850         TAILQ_ENTRY(meter_profile) node;
 851         char name[RTE_SWX_NAME_SIZE];
 852         struct rte_meter_trtcm_params params;
 853         struct rte_meter_trtcm_profile profile;
 854         uint32_t n_users;
 855 };
 856
 857 TAILQ_HEAD(meter_profile_tailq, meter_profile);
 858
 859 struct metarray {
 860         TAILQ_ENTRY(metarray) node;
 861         char name[RTE_SWX_NAME_SIZE];
 862         uint32_t size;
 863         uint32_t id;
 864 };
 865
 866 TAILQ_HEAD(metarray_tailq, metarray);
 867
 868 struct meter {
 869         struct rte_meter_trtcm m;
 870         struct meter_profile *profile;
 871         enum rte_color color_mask;
 872         uint8_t pad[20];
 873
 874         uint64_t n_pkts[RTE_COLORS];
 875         uint64_t n_bytes[RTE_COLORS];
 876 };
 877
 878 struct metarray_runtime {
 879         struct meter *metarray;
 880         uint32_t size_mask;
 881 };
 882
 883 /*
 884  * Pipeline.
 885  */
 886 struct thread {
 887         /* Packet. */
 888         struct rte_swx_pkt pkt;
 889         uint8_t *ptr;
 890
 891         /* Structures. */
 892         uint8_t **structs;
 893
 894         /* Packet headers. */
 895         struct header_runtime *headers; /* Extracted or generated headers. */
 896         struct header_out_runtime *headers_out; /* Emitted headers. */
 897         uint8_t *header_storage;
 898         uint8_t *header_out_storage;
 899         uint64_t valid_headers;
 900         uint32_t n_headers_out;
 901
 902         /* Packet meta-data. */
 903         uint8_t *metadata;
 904
 905         /* Tables. */
 906         struct table_runtime *tables;
 907         struct selector_runtime *selectors;
 908         struct learner_runtime *learners;
 909         struct rte_swx_table_state *table_state;
 910         uint64_t action_id;
 911         int hit; /* 0 = Miss, 1 = Hit. */
 912         uint32_t learner_id;
 913         uint64_t time;
 914
 915         /* Extern objects and functions. */
 916         struct extern_obj_runtime *extern_objs;
 917         struct extern_func_runtime *extern_funcs;
 918
 919         /* Instructions. */
 920         struct instruction *ip;
 921         struct instruction *ret;
 922 };
 923
 924 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 925 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 926 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 927
 928 #define HEADER_VALID(thread, header_id) \
 929         MASK64_BIT_GET((thread)->valid_headers, header_id)
 930
 931 #define ALU(thread, ip, operator)  \
 932 {                                                                              \
 933         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 934         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 935         uint64_t dst64 = *dst64_ptr;                                           \
 936         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 937         uint64_t dst = dst64 & dst64_mask;                                     \
 938                                                                                \
 939         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 940         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 941         uint64_t src64 = *src64_ptr;                                           \
 942         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 943         uint64_t src = src64 & src64_mask;                                     \
 944                                                                                \
 945         uint64_t result = dst operator src;                                    \
 946                                                                                \
 947         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 948 }
 949
 950 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 951
 952 #define ALU_MH(thread, ip, operator)  \
 953 {                                                                              \
 954         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 955         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 956         uint64_t dst64 = *dst64_ptr;                                           \
 957         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 958         uint64_t dst = dst64 & dst64_mask;                                     \
 959                                                                                \
 960         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 961         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 962         uint64_t src64 = *src64_ptr;                                           \
 963         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 964                                                                                \
 965         uint64_t result = dst operator src;                                    \
 966                                                                                \
 967         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 968 }
 969
 970 #define ALU_HM(thread, ip, operator)  \
 971 {                                                                              \
 972         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 973         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 974         uint64_t dst64 = *dst64_ptr;                                           \
 975         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 976         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 977                                                                                \
 978         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 979         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 980         uint64_t src64 = *src64_ptr;                                           \
 981         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 982         uint64_t src = src64 & src64_mask;                                     \
 983                                                                                \
 984         uint64_t result = dst operator src;                                    \
 985         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 986                                                                                \
 987         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 988 }
 989
 990 #define ALU_HM_FAST(thread, ip, operator)  \
 991 {                                                                                 \
 992         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];         \
 993         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];      \
 994         uint64_t dst64 = *dst64_ptr;                                              \
 995         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);          \
 996         uint64_t dst = dst64 & dst64_mask;                                        \
 997                                                                                   \
 998         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];         \
 999         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];      \
1000         uint64_t src64 = *src64_ptr;                                              \
1001         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);          \
1002         uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1003                                                                                   \
1004         uint64_t result = dst operator src;                                       \
1005                                                                                   \
1006         *dst64_ptr = (dst64 & ~dst64_mask) | result;                              \
1007 }
1008
1009 #define ALU_HH(thread, ip, operator)  \
1010 {                                                                              \
1011         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1012         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1013         uint64_t dst64 = *dst64_ptr;                                           \
1014         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1015         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
1016                                                                                \
1017         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
1018         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
1019         uint64_t src64 = *src64_ptr;                                           \
1020         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
1021                                                                                \
1022         uint64_t result = dst operator src;                                    \
1023         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1024                                                                                \
1025         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1026 }
1027
1028 #define ALU_HH_FAST(thread, ip, operator)  \
1029 {                                                                                             \
1030         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];                     \
1031         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];                  \
1032         uint64_t dst64 = *dst64_ptr;                                                          \
1033         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);                      \
1034         uint64_t dst = dst64 & dst64_mask;                                                    \
1035                                                                                               \
1036         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];                     \
1037         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];                  \
1038         uint64_t src64 = *src64_ptr;                                                          \
1039         uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1040                                                                                               \
1041         uint64_t result = dst operator src;                                                   \
1042                                                                                               \
1043         *dst64_ptr = (dst64 & ~dst64_mask) | result;                                          \
1044 }
1045
1046 #else
1047
1048 #define ALU_MH ALU
1049 #define ALU_HM ALU
1050 #define ALU_HM_FAST ALU
1051 #define ALU_HH ALU
1052 #define ALU_HH_FAST ALU
1053
1054 #endif
1055
1056 #define ALU_I(thread, ip, operator)  \
1057 {                                                                              \
1058         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1059         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1060         uint64_t dst64 = *dst64_ptr;                                           \
1061         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1062         uint64_t dst = dst64 & dst64_mask;                                     \
1063                                                                                \
1064         uint64_t src = (ip)->alu.src_val;                                      \
1065                                                                                \
1066         uint64_t result = dst operator src;                                    \
1067                                                                                \
1068         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
1069 }
1070
1071 #define ALU_MI ALU_I
1072
1073 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1074
1075 #define ALU_HI(thread, ip, operator)  \
1076 {                                                                              \
1077         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
1078         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
1079         uint64_t dst64 = *dst64_ptr;                                           \
1080         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
1081         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
1082                                                                                \
1083         uint64_t src = (ip)->alu.src_val;                                      \
1084                                                                                \
1085         uint64_t result = dst operator src;                                    \
1086         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
1087                                                                                \
1088         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
1089 }
1090
1091 #else
1092
1093 #define ALU_HI ALU_I
1094
1095 #endif
1096
1097 #define MOV(thread, ip)  \
1098 {                                                                              \
1099         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1100         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1101         uint64_t dst64 = *dst64_ptr;                                           \
1102         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1103                                                                                \
1104         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1105         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1106         uint64_t src64 = *src64_ptr;                                           \
1107         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1108         uint64_t src = src64 & src64_mask;                                     \
1109                                                                                \
1110         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1111 }
1112
1113 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1114
1115 #define MOV_MH(thread, ip)  \
1116 {                                                                              \
1117         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1118         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1119         uint64_t dst64 = *dst64_ptr;                                           \
1120         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1121                                                                                \
1122         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1123         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1124         uint64_t src64 = *src64_ptr;                                           \
1125         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
1126                                                                                \
1127         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1128 }
1129
1130 #define MOV_HM(thread, ip)  \
1131 {                                                                              \
1132         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1133         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1134         uint64_t dst64 = *dst64_ptr;                                           \
1135         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1136                                                                                \
1137         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1138         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1139         uint64_t src64 = *src64_ptr;                                           \
1140         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
1141         uint64_t src = src64 & src64_mask;                                     \
1142                                                                                \
1143         src = hton64(src) >> (64 - (ip)->mov.dst.n_bits);                      \
1144         *dst64_ptr = (dst64 & ~dst64_mask) | src;                              \
1145 }
1146
1147 #define MOV_HH(thread, ip)  \
1148 {                                                                              \
1149         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1150         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1151         uint64_t dst64 = *dst64_ptr;                                           \
1152         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1153                                                                                \
1154         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
1155         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
1156         uint64_t src64 = *src64_ptr;                                           \
1157                                                                                \
1158         uint64_t src = src64 << (64 - (ip)->mov.src.n_bits);                   \
1159         src = src >> (64 - (ip)->mov.dst.n_bits);                              \
1160         *dst64_ptr = (dst64 & ~dst64_mask) | src;                              \
1161 }
1162
1163 #else
1164
1165 #define MOV_MH MOV
1166 #define MOV_HM MOV
1167 #define MOV_HH MOV
1168
1169 #endif
1170
1171 #define MOV_I(thread, ip)  \
1172 {                                                                              \
1173         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
1174         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
1175         uint64_t dst64 = *dst64_ptr;                                           \
1176         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
1177                                                                                \
1178         uint64_t src = (ip)->mov.src_val;                                      \
1179                                                                                \
1180         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
1181 }
1182
1183 #define JMP_CMP(thread, ip, operator)  \
1184 {                                                                              \
1185         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1186         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1187         uint64_t a64 = *a64_ptr;                                               \
1188         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1189         uint64_t a = a64 & a64_mask;                                           \
1190                                                                                \
1191         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1192         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1193         uint64_t b64 = *b64_ptr;                                               \
1194         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1195         uint64_t b = b64 & b64_mask;                                           \
1196                                                                                \
1197         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1198 }
1199
1200 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1201
1202 #define JMP_CMP_MH(thread, ip, operator)  \
1203 {                                                                              \
1204         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1205         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1206         uint64_t a64 = *a64_ptr;                                               \
1207         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1208         uint64_t a = a64 & a64_mask;                                           \
1209                                                                                \
1210         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1211         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1212         uint64_t b64 = *b64_ptr;                                               \
1213         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1214                                                                                \
1215         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1216 }
1217
1218 #define JMP_CMP_HM(thread, ip, operator)  \
1219 {                                                                              \
1220         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1221         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1222         uint64_t a64 = *a64_ptr;                                               \
1223         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1224                                                                                \
1225         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1226         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1227         uint64_t b64 = *b64_ptr;                                               \
1228         uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
1229         uint64_t b = b64 & b64_mask;                                           \
1230                                                                                \
1231         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1232 }
1233
1234 #define JMP_CMP_HH(thread, ip, operator)  \
1235 {                                                                              \
1236         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1237         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1238         uint64_t a64 = *a64_ptr;                                               \
1239         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1240                                                                                \
1241         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1242         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1243         uint64_t b64 = *b64_ptr;                                               \
1244         uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
1245                                                                                \
1246         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1247 }
1248
1249 #define JMP_CMP_HH_FAST(thread, ip, operator)  \
1250 {                                                                              \
1251         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1252         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1253         uint64_t a64 = *a64_ptr;                                               \
1254         uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits);                         \
1255                                                                                \
1256         uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
1257         uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
1258         uint64_t b64 = *b64_ptr;                                               \
1259         uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits);                         \
1260                                                                                \
1261         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1262 }
1263
1264 #else
1265
1266 #define JMP_CMP_MH JMP_CMP
1267 #define JMP_CMP_HM JMP_CMP
1268 #define JMP_CMP_HH JMP_CMP
1269 #define JMP_CMP_HH_FAST JMP_CMP
1270
1271 #endif
1272
1273 #define JMP_CMP_I(thread, ip, operator)  \
1274 {                                                                              \
1275         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1276         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1277         uint64_t a64 = *a64_ptr;                                               \
1278         uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
1279         uint64_t a = a64 & a64_mask;                                           \
1280                                                                                \
1281         uint64_t b = (ip)->jmp.b_val;                                          \
1282                                                                                \
1283         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1284 }
1285
1286 #define JMP_CMP_MI JMP_CMP_I
1287
1288 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1289
1290 #define JMP_CMP_HI(thread, ip, operator)  \
1291 {                                                                              \
1292         uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
1293         uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
1294         uint64_t a64 = *a64_ptr;                                               \
1295         uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
1296                                                                                \
1297         uint64_t b = (ip)->jmp.b_val;                                          \
1298                                                                                \
1299         (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
1300 }
1301
1302 #else
1303
1304 #define JMP_CMP_HI JMP_CMP_I
1305
1306 #endif
1307
1308 #define METADATA_READ(thread, offset, n_bits)                                  \
1309 ({                                                                             \
1310         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1311         uint64_t m64 = *m64_ptr;                                               \
1312         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1313         (m64 & m64_mask);                                                      \
1314 })
1315
1316 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
1317 {                                                                              \
1318         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
1319         uint64_t m64 = *m64_ptr;                                               \
1320         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
1321                                                                                \
1322         uint64_t m_new = value;                                                \
1323                                                                                \
1324         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
1325 }
1326
1327 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1328 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1329 #endif
1330
1331 struct rte_swx_pipeline {
1332         struct struct_type_tailq struct_types;
1333         struct port_in_type_tailq port_in_types;
1334         struct port_in_tailq ports_in;
1335         struct port_out_type_tailq port_out_types;
1336         struct port_out_tailq ports_out;
1337         struct extern_type_tailq extern_types;
1338         struct extern_obj_tailq extern_objs;
1339         struct extern_func_tailq extern_funcs;
1340         struct header_tailq headers;
1341         struct struct_type *metadata_st;
1342         uint32_t metadata_struct_id;
1343         struct action_tailq actions;
1344         struct table_type_tailq table_types;
1345         struct table_tailq tables;
1346         struct selector_tailq selectors;
1347         struct learner_tailq learners;
1348         struct regarray_tailq regarrays;
1349         struct meter_profile_tailq meter_profiles;
1350         struct metarray_tailq metarrays;
1351
1352         struct port_in_runtime *in;
1353         struct port_out_runtime *out;
1354         struct instruction **action_instructions;
1355         struct rte_swx_table_state *table_state;
1356         struct table_statistics *table_stats;
1357         struct selector_statistics *selector_stats;
1358         struct learner_statistics *learner_stats;
1359         struct regarray_runtime *regarray_runtime;
1360         struct metarray_runtime *metarray_runtime;
1361         struct instruction *instructions;
1362         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1363
1364         uint32_t n_structs;
1365         uint32_t n_ports_in;
1366         uint32_t n_ports_out;
1367         uint32_t n_extern_objs;
1368         uint32_t n_extern_funcs;
1369         uint32_t n_actions;
1370         uint32_t n_tables;
1371         uint32_t n_selectors;
1372         uint32_t n_learners;
1373         uint32_t n_regarrays;
1374         uint32_t n_metarrays;
1375         uint32_t n_headers;
1376         uint32_t thread_id;
1377         uint32_t port_id;
1378         uint32_t n_instructions;
1379         int build_done;
1380         int numa_node;
1381 };
1382
1383 /*
1384  * Instruction.
1385  */
1386 static inline void
1387 pipeline_port_inc(struct rte_swx_pipeline *p)
1388 {
1389         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
1390 }
1391
1392 static inline void
1393 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1394 {
1395         t->ip = p->instructions;
1396 }
1397
1398 static inline void
1399 thread_ip_set(struct thread *t, struct instruction *ip)
1400 {
1401         t->ip = ip;
1402 }
1403
1404 static inline void
1405 thread_ip_action_call(struct rte_swx_pipeline *p,
1406                       struct thread *t,
1407                       uint32_t action_id)
1408 {
1409         t->ret = t->ip + 1;
1410         t->ip = p->action_instructions[action_id];
1411 }
1412
1413 static inline void
1414 thread_ip_inc(struct rte_swx_pipeline *p);
1415
1416 static inline void
1417 thread_ip_inc(struct rte_swx_pipeline *p)
1418 {
1419         struct thread *t = &p->threads[p->thread_id];
1420
1421         t->ip++;
1422 }
1423
1424 static inline void
1425 thread_ip_inc_cond(struct thread *t, int cond)
1426 {
1427         t->ip += cond;
1428 }
1429
1430 static inline void
1431 thread_yield(struct rte_swx_pipeline *p)
1432 {
1433         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1434 }
1435
1436 static inline void
1437 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
1438 {
1439         p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1440 }
1441
1442 /*
1443  * rx.
1444  */
1445 static inline int
1446 __instr_rx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1447 {
1448         struct port_in_runtime *port = &p->in[p->port_id];
1449         struct rte_swx_pkt *pkt = &t->pkt;
1450         int pkt_received;
1451
1452         /* Packet. */
1453         pkt_received = port->pkt_rx(port->obj, pkt);
1454         t->ptr = &pkt->pkt[pkt->offset];
1455         rte_prefetch0(t->ptr);
1456
1457         TRACE("[Thread %2u] rx %s from port %u\n",
1458               p->thread_id,
1459               pkt_received ? "1 pkt" : "0 pkts",
1460               p->port_id);
1461
1462         /* Headers. */
1463         t->valid_headers = 0;
1464         t->n_headers_out = 0;
1465
1466         /* Meta-data. */
1467         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1468
1469         /* Tables. */
1470         t->table_state = p->table_state;
1471
1472         /* Thread. */
1473         pipeline_port_inc(p);
1474
1475         return pkt_received;
1476 }
1477
1478 static inline void
1479 instr_rx_exec(struct rte_swx_pipeline *p)
1480 {
1481         struct thread *t = &p->threads[p->thread_id];
1482         struct instruction *ip = t->ip;
1483         int pkt_received;
1484
1485         /* Packet. */
1486         pkt_received = __instr_rx_exec(p, t, ip);
1487
1488         /* Thread. */
1489         thread_ip_inc_cond(t, pkt_received);
1490         thread_yield(p);
1491 }
1492
1493 #endif