lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9 #include <arpa/inet.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_prefetch.h>
  13 #include <rte_byteorder.h>
  14
  15 #include "rte_swx_pipeline.h"
  16 #include "rte_swx_ctl.h"
  17
  18 #define CHECK(condition, err_code)                                             \
  19 do {                                                                           \
  20         if (!(condition))                                                      \
  21                 return -(err_code);                                            \
  22 } while (0)
  23
  24 #define CHECK_NAME(name, err_code)                                             \
  25         CHECK((name) && (name)[0], err_code)
  26
  27 #ifndef TRACE_LEVEL
  28 #define TRACE_LEVEL 0
  29 #endif
  30
  31 #if TRACE_LEVEL
  32 #define TRACE(...) printf(__VA_ARGS__)
  33 #else
  34 #define TRACE(...)
  35 #endif
  36
  37 #define ntoh64(x) rte_be_to_cpu_64(x)
  38 #define hton64(x) rte_cpu_to_be_64(x)
  39
  40 /*
  41  * Struct.
  42  */
  43 struct field {
  44         char name[RTE_SWX_NAME_SIZE];
  45         uint32_t n_bits;
  46         uint32_t offset;
  47 };
  48
  49 struct struct_type {
  50         TAILQ_ENTRY(struct_type) node;
  51         char name[RTE_SWX_NAME_SIZE];
  52         struct field *fields;
  53         uint32_t n_fields;
  54         uint32_t n_bits;
  55 };
  56
  57 TAILQ_HEAD(struct_type_tailq, struct_type);
  58
  59 /*
  60  * Input port.
  61  */
  62 struct port_in_type {
  63         TAILQ_ENTRY(port_in_type) node;
  64         char name[RTE_SWX_NAME_SIZE];
  65         struct rte_swx_port_in_ops ops;
  66 };
  67
  68 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  69
  70 struct port_in {
  71         TAILQ_ENTRY(port_in) node;
  72         struct port_in_type *type;
  73         void *obj;
  74         uint32_t id;
  75 };
  76
  77 TAILQ_HEAD(port_in_tailq, port_in);
  78
  79 struct port_in_runtime {
  80         rte_swx_port_in_pkt_rx_t pkt_rx;
  81         void *obj;
  82 };
  83
  84 /*
  85  * Output port.
  86  */
  87 struct port_out_type {
  88         TAILQ_ENTRY(port_out_type) node;
  89         char name[RTE_SWX_NAME_SIZE];
  90         struct rte_swx_port_out_ops ops;
  91 };
  92
  93 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  94
  95 struct port_out {
  96         TAILQ_ENTRY(port_out) node;
  97         struct port_out_type *type;
  98         void *obj;
  99         uint32_t id;
 100 };
 101
 102 TAILQ_HEAD(port_out_tailq, port_out);
 103
 104 struct port_out_runtime {
 105         rte_swx_port_out_pkt_tx_t pkt_tx;
 106         rte_swx_port_out_flush_t flush;
 107         void *obj;
 108 };
 109
 110 /*
 111  * Extern object.
 112  */
 113 struct extern_type_member_func {
 114         TAILQ_ENTRY(extern_type_member_func) node;
 115         char name[RTE_SWX_NAME_SIZE];
 116         rte_swx_extern_type_member_func_t func;
 117         uint32_t id;
 118 };
 119
 120 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 121
 122 struct extern_type {
 123         TAILQ_ENTRY(extern_type) node;
 124         char name[RTE_SWX_NAME_SIZE];
 125         struct struct_type *mailbox_struct_type;
 126         rte_swx_extern_type_constructor_t constructor;
 127         rte_swx_extern_type_destructor_t destructor;
 128         struct extern_type_member_func_tailq funcs;
 129         uint32_t n_funcs;
 130 };
 131
 132 TAILQ_HEAD(extern_type_tailq, extern_type);
 133
 134 struct extern_obj {
 135         TAILQ_ENTRY(extern_obj) node;
 136         char name[RTE_SWX_NAME_SIZE];
 137         struct extern_type *type;
 138         void *obj;
 139         uint32_t struct_id;
 140         uint32_t id;
 141 };
 142
 143 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 144
 145 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 146 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 147 #endif
 148
 149 struct extern_obj_runtime {
 150         void *obj;
 151         uint8_t *mailbox;
 152         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 153 };
 154
 155 /*
 156  * Extern function.
 157  */
 158 struct extern_func {
 159         TAILQ_ENTRY(extern_func) node;
 160         char name[RTE_SWX_NAME_SIZE];
 161         struct struct_type *mailbox_struct_type;
 162         rte_swx_extern_func_t func;
 163         uint32_t struct_id;
 164         uint32_t id;
 165 };
 166
 167 TAILQ_HEAD(extern_func_tailq, extern_func);
 168
 169 struct extern_func_runtime {
 170         uint8_t *mailbox;
 171         rte_swx_extern_func_t func;
 172 };
 173
 174 /*
 175  * Header.
 176  */
 177 struct header {
 178         TAILQ_ENTRY(header) node;
 179         char name[RTE_SWX_NAME_SIZE];
 180         struct struct_type *st;
 181         uint32_t struct_id;
 182         uint32_t id;
 183 };
 184
 185 TAILQ_HEAD(header_tailq, header);
 186
 187 struct header_runtime {
 188         uint8_t *ptr0;
 189 };
 190
 191 struct header_out_runtime {
 192         uint8_t *ptr0;
 193         uint8_t *ptr;
 194         uint32_t n_bytes;
 195 };
 196
 197 /*
 198  * Instruction.
 199  */
 200
 201 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 202  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 203  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 204  * when transferred to packet meta-data and in NBO when transferred to packet
 205  * headers.
 206  */
 207
 208 /* Notation conventions:
 209  *    -Header field: H = h.header.field (dst/src)
 210  *    -Meta-data field: M = m.field (dst/src)
 211  *    -Extern object mailbox field: E = e.field (dst/src)
 212  *    -Extern function mailbox field: F = f.field (dst/src)
 213  *    -Table action data field: T = t.field (src only)
 214  *    -Immediate value: I = 32-bit unsigned value (src only)
 215  */
 216
 217 enum instruction_type {
 218         /* rx m.port_in */
 219         INSTR_RX,
 220
 221         /* tx m.port_out */
 222         INSTR_TX,
 223
 224         /* extract h.header */
 225         INSTR_HDR_EXTRACT,
 226         INSTR_HDR_EXTRACT2,
 227         INSTR_HDR_EXTRACT3,
 228         INSTR_HDR_EXTRACT4,
 229         INSTR_HDR_EXTRACT5,
 230         INSTR_HDR_EXTRACT6,
 231         INSTR_HDR_EXTRACT7,
 232         INSTR_HDR_EXTRACT8,
 233
 234         /* emit h.header */
 235         INSTR_HDR_EMIT,
 236         INSTR_HDR_EMIT_TX,
 237         INSTR_HDR_EMIT2_TX,
 238         INSTR_HDR_EMIT3_TX,
 239         INSTR_HDR_EMIT4_TX,
 240         INSTR_HDR_EMIT5_TX,
 241         INSTR_HDR_EMIT6_TX,
 242         INSTR_HDR_EMIT7_TX,
 243         INSTR_HDR_EMIT8_TX,
 244
 245         /* validate h.header */
 246         INSTR_HDR_VALIDATE,
 247
 248         /* invalidate h.header */
 249         INSTR_HDR_INVALIDATE,
 250
 251         /* mov dst src
 252          * dst = src
 253          * dst = HMEF, src = HMEFTI
 254          */
 255         INSTR_MOV,   /* dst = MEF, src = MEFT */
 256         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 257         INSTR_MOV_I, /* dst = HMEF, src = I */
 258
 259         /* dma h.header t.field
 260          * memcpy(h.header, t.field, sizeof(h.header))
 261          */
 262         INSTR_DMA_HT,
 263         INSTR_DMA_HT2,
 264         INSTR_DMA_HT3,
 265         INSTR_DMA_HT4,
 266         INSTR_DMA_HT5,
 267         INSTR_DMA_HT6,
 268         INSTR_DMA_HT7,
 269         INSTR_DMA_HT8,
 270
 271         /* add dst src
 272          * dst += src
 273          * dst = HMEF, src = HMEFTI
 274          */
 275         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 276         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 277         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 278         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 279         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 280         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 281
 282         /* sub dst src
 283          * dst -= src
 284          * dst = HMEF, src = HMEFTI
 285          */
 286         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 287         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 288         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 289         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 290         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 291         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 292
 293         /* ckadd dst src
 294          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 295          * dst = H, src = {H, h.header}
 296          */
 297         INSTR_ALU_CKADD_FIELD,    /* src = H */
 298         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 299         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 300
 301         /* cksub dst src
 302          * dst = dst '- src
 303          * dst = H, src = H
 304          */
 305         INSTR_ALU_CKSUB_FIELD,
 306
 307         /* and dst src
 308          * dst &= src
 309          * dst = HMEF, src = HMEFTI
 310          */
 311         INSTR_ALU_AND,   /* dst = MEF, src = MEFT */
 312         INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 313         INSTR_ALU_AND_I, /* dst = HMEF, src = I */
 314
 315         /* or dst src
 316          * dst |= src
 317          * dst = HMEF, src = HMEFTI
 318          */
 319         INSTR_ALU_OR,   /* dst = MEF, src = MEFT */
 320         INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 321         INSTR_ALU_OR_I, /* dst = HMEF, src = I */
 322
 323         /* xor dst src
 324          * dst ^= src
 325          * dst = HMEF, src = HMEFTI
 326          */
 327         INSTR_ALU_XOR,   /* dst = MEF, src = MEFT */
 328         INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 329         INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
 330
 331         /* shl dst src
 332          * dst <<= src
 333          * dst = HMEF, src = HMEFTI
 334          */
 335         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 336         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 337         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 338         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 339         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 340         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 341
 342         /* shr dst src
 343          * dst >>= src
 344          * dst = HMEF, src = HMEFTI
 345          */
 346         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 347         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 348         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 349         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 350         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 351         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 352 };
 353
 354 struct instr_operand {
 355         uint8_t struct_id;
 356         uint8_t n_bits;
 357         uint8_t offset;
 358         uint8_t pad;
 359 };
 360
 361 struct instr_io {
 362         struct {
 363                 uint8_t offset;
 364                 uint8_t n_bits;
 365                 uint8_t pad[2];
 366         } io;
 367
 368         struct {
 369                 uint8_t header_id[8];
 370                 uint8_t struct_id[8];
 371                 uint8_t n_bytes[8];
 372         } hdr;
 373 };
 374
 375 struct instr_hdr_validity {
 376         uint8_t header_id;
 377 };
 378
 379 struct instr_dst_src {
 380         struct instr_operand dst;
 381         union {
 382                 struct instr_operand src;
 383                 uint32_t src_val;
 384         };
 385 };
 386
 387 struct instr_dma {
 388         struct {
 389                 uint8_t header_id[8];
 390                 uint8_t struct_id[8];
 391         } dst;
 392
 393         struct {
 394                 uint8_t offset[8];
 395         } src;
 396
 397         uint16_t n_bytes[8];
 398 };
 399
 400 struct instruction {
 401         enum instruction_type type;
 402         union {
 403                 struct instr_io io;
 404                 struct instr_hdr_validity valid;
 405                 struct instr_dst_src mov;
 406                 struct instr_dma dma;
 407                 struct instr_dst_src alu;
 408         };
 409 };
 410
 411 struct instruction_data {
 412         char label[RTE_SWX_NAME_SIZE];
 413         char jmp_label[RTE_SWX_NAME_SIZE];
 414         uint32_t n_users; /* user = jmp instruction to this instruction. */
 415         int invalid;
 416 };
 417
 418 /*
 419  * Action.
 420  */
 421 struct action {
 422         TAILQ_ENTRY(action) node;
 423         char name[RTE_SWX_NAME_SIZE];
 424         struct struct_type *st;
 425         struct instruction *instructions;
 426         uint32_t n_instructions;
 427         uint32_t id;
 428 };
 429
 430 TAILQ_HEAD(action_tailq, action);
 431
 432 /*
 433  * Table.
 434  */
 435 struct table_type {
 436         TAILQ_ENTRY(table_type) node;
 437         char name[RTE_SWX_NAME_SIZE];
 438         enum rte_swx_table_match_type match_type;
 439         struct rte_swx_table_ops ops;
 440 };
 441
 442 TAILQ_HEAD(table_type_tailq, table_type);
 443
 444 struct match_field {
 445         enum rte_swx_table_match_type match_type;
 446         struct field *field;
 447 };
 448
 449 struct table {
 450         TAILQ_ENTRY(table) node;
 451         char name[RTE_SWX_NAME_SIZE];
 452         char args[RTE_SWX_NAME_SIZE];
 453         struct table_type *type; /* NULL when n_fields == 0. */
 454
 455         /* Match. */
 456         struct match_field *fields;
 457         uint32_t n_fields;
 458         int is_header; /* Only valid when n_fields > 0. */
 459         struct header *header; /* Only valid when n_fields > 0. */
 460
 461         /* Action. */
 462         struct action **actions;
 463         struct action *default_action;
 464         uint8_t *default_action_data;
 465         uint32_t n_actions;
 466         int default_action_is_const;
 467         uint32_t action_data_size_max;
 468
 469         uint32_t size;
 470         uint32_t id;
 471 };
 472
 473 TAILQ_HEAD(table_tailq, table);
 474
 475 struct table_runtime {
 476         rte_swx_table_lookup_t func;
 477         void *mailbox;
 478         uint8_t **key;
 479 };
 480
 481 /*
 482  * Pipeline.
 483  */
 484 struct thread {
 485         /* Packet. */
 486         struct rte_swx_pkt pkt;
 487         uint8_t *ptr;
 488
 489         /* Structures. */
 490         uint8_t **structs;
 491
 492         /* Packet headers. */
 493         struct header_runtime *headers; /* Extracted or generated headers. */
 494         struct header_out_runtime *headers_out; /* Emitted headers. */
 495         uint8_t *header_storage;
 496         uint8_t *header_out_storage;
 497         uint64_t valid_headers;
 498         uint32_t n_headers_out;
 499
 500         /* Packet meta-data. */
 501         uint8_t *metadata;
 502
 503         /* Tables. */
 504         struct table_runtime *tables;
 505         struct rte_swx_table_state *table_state;
 506         uint64_t action_id;
 507         int hit; /* 0 = Miss, 1 = Hit. */
 508
 509         /* Extern objects and functions. */
 510         struct extern_obj_runtime *extern_objs;
 511         struct extern_func_runtime *extern_funcs;
 512
 513         /* Instructions. */
 514         struct instruction *ip;
 515         struct instruction *ret;
 516 };
 517
 518 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 519 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 520 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 521
 522 #define ALU(thread, ip, operator)  \
 523 {                                                                              \
 524         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 525         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 526         uint64_t dst64 = *dst64_ptr;                                           \
 527         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 528         uint64_t dst = dst64 & dst64_mask;                                     \
 529                                                                                \
 530         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 531         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 532         uint64_t src64 = *src64_ptr;                                           \
 533         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 534         uint64_t src = src64 & src64_mask;                                     \
 535                                                                                \
 536         uint64_t result = dst operator src;                                    \
 537                                                                                \
 538         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 539 }
 540
 541 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 542
 543 #define ALU_S(thread, ip, operator)  \
 544 {                                                                              \
 545         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 546         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 547         uint64_t dst64 = *dst64_ptr;                                           \
 548         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 549         uint64_t dst = dst64 & dst64_mask;                                     \
 550                                                                                \
 551         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 552         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 553         uint64_t src64 = *src64_ptr;                                           \
 554         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 555                                                                                \
 556         uint64_t result = dst operator src;                                    \
 557                                                                                \
 558         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 559 }
 560
 561 #define ALU_MH ALU_S
 562
 563 #define ALU_HM(thread, ip, operator)  \
 564 {                                                                              \
 565         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 566         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 567         uint64_t dst64 = *dst64_ptr;                                           \
 568         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 569         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 570                                                                                \
 571         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 572         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 573         uint64_t src64 = *src64_ptr;                                           \
 574         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 575         uint64_t src = src64 & src64_mask;                                     \
 576                                                                                \
 577         uint64_t result = dst operator src;                                    \
 578         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 579                                                                                \
 580         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 581 }
 582
 583 #define ALU_HH(thread, ip, operator)  \
 584 {                                                                              \
 585         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 586         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 587         uint64_t dst64 = *dst64_ptr;                                           \
 588         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 589         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 590                                                                                \
 591         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 592         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 593         uint64_t src64 = *src64_ptr;                                           \
 594         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 595                                                                                \
 596         uint64_t result = dst operator src;                                    \
 597         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 598                                                                                \
 599         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 600 }
 601
 602 #else
 603
 604 #define ALU_S ALU
 605 #define ALU_MH ALU
 606 #define ALU_HM ALU
 607 #define ALU_HH ALU
 608
 609 #endif
 610
 611 #define ALU_I(thread, ip, operator)  \
 612 {                                                                              \
 613         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 614         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 615         uint64_t dst64 = *dst64_ptr;                                           \
 616         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 617         uint64_t dst = dst64 & dst64_mask;                                     \
 618                                                                                \
 619         uint64_t src = (ip)->alu.src_val;                                      \
 620                                                                                \
 621         uint64_t result = dst operator src;                                    \
 622                                                                                \
 623         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 624 }
 625
 626 #define ALU_MI ALU_I
 627
 628 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 629
 630 #define ALU_HI(thread, ip, operator)  \
 631 {                                                                              \
 632         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 633         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 634         uint64_t dst64 = *dst64_ptr;                                           \
 635         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 636         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 637                                                                                \
 638         uint64_t src = (ip)->alu.src_val;                                      \
 639                                                                                \
 640         uint64_t result = dst operator src;                                    \
 641         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 642                                                                                \
 643         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 644 }
 645
 646 #else
 647
 648 #define ALU_HI ALU_I
 649
 650 #endif
 651
 652 #define MOV(thread, ip)  \
 653 {                                                                              \
 654         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 655         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 656         uint64_t dst64 = *dst64_ptr;                                           \
 657         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 658                                                                                \
 659         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 660         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 661         uint64_t src64 = *src64_ptr;                                           \
 662         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
 663         uint64_t src = src64 & src64_mask;                                     \
 664                                                                                \
 665         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 666 }
 667
 668 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 669
 670 #define MOV_S(thread, ip)  \
 671 {                                                                              \
 672         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 673         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 674         uint64_t dst64 = *dst64_ptr;                                           \
 675         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 676                                                                                \
 677         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 678         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 679         uint64_t src64 = *src64_ptr;                                           \
 680         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
 681                                                                                \
 682         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 683 }
 684
 685 #else
 686
 687 #define MOV_S MOV
 688
 689 #endif
 690
 691 #define MOV_I(thread, ip)  \
 692 {                                                                              \
 693         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 694         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 695         uint64_t dst64 = *dst64_ptr;                                           \
 696         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 697                                                                                \
 698         uint64_t src = (ip)->mov.src_val;                                      \
 699                                                                                \
 700         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 701 }
 702
 703 #define METADATA_READ(thread, offset, n_bits)                                  \
 704 ({                                                                             \
 705         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 706         uint64_t m64 = *m64_ptr;                                               \
 707         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 708         (m64 & m64_mask);                                                      \
 709 })
 710
 711 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
 712 {                                                                              \
 713         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 714         uint64_t m64 = *m64_ptr;                                               \
 715         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 716                                                                                \
 717         uint64_t m_new = value;                                                \
 718                                                                                \
 719         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
 720 }
 721
 722 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 723 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 724 #endif
 725
 726 struct rte_swx_pipeline {
 727         struct struct_type_tailq struct_types;
 728         struct port_in_type_tailq port_in_types;
 729         struct port_in_tailq ports_in;
 730         struct port_out_type_tailq port_out_types;
 731         struct port_out_tailq ports_out;
 732         struct extern_type_tailq extern_types;
 733         struct extern_obj_tailq extern_objs;
 734         struct extern_func_tailq extern_funcs;
 735         struct header_tailq headers;
 736         struct struct_type *metadata_st;
 737         uint32_t metadata_struct_id;
 738         struct action_tailq actions;
 739         struct table_type_tailq table_types;
 740         struct table_tailq tables;
 741
 742         struct port_in_runtime *in;
 743         struct port_out_runtime *out;
 744         struct instruction **action_instructions;
 745         struct rte_swx_table_state *table_state;
 746         struct instruction *instructions;
 747         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 748
 749         uint32_t n_structs;
 750         uint32_t n_ports_in;
 751         uint32_t n_ports_out;
 752         uint32_t n_extern_objs;
 753         uint32_t n_extern_funcs;
 754         uint32_t n_actions;
 755         uint32_t n_tables;
 756         uint32_t n_headers;
 757         uint32_t thread_id;
 758         uint32_t port_id;
 759         uint32_t n_instructions;
 760         int build_done;
 761         int numa_node;
 762 };
 763
 764 /*
 765  * Struct.
 766  */
 767 static struct struct_type *
 768 struct_type_find(struct rte_swx_pipeline *p, const char *name)
 769 {
 770         struct struct_type *elem;
 771
 772         TAILQ_FOREACH(elem, &p->struct_types, node)
 773                 if (strcmp(elem->name, name) == 0)
 774                         return elem;
 775
 776         return NULL;
 777 }
 778
 779 static struct field *
 780 struct_type_field_find(struct struct_type *st, const char *name)
 781 {
 782         uint32_t i;
 783
 784         for (i = 0; i < st->n_fields; i++) {
 785                 struct field *f = &st->fields[i];
 786
 787                 if (strcmp(f->name, name) == 0)
 788                         return f;
 789         }
 790
 791         return NULL;
 792 }
 793
 794 int
 795 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 796                                       const char *name,
 797                                       struct rte_swx_field_params *fields,
 798                                       uint32_t n_fields)
 799 {
 800         struct struct_type *st;
 801         uint32_t i;
 802
 803         CHECK(p, EINVAL);
 804         CHECK_NAME(name, EINVAL);
 805         CHECK(fields, EINVAL);
 806         CHECK(n_fields, EINVAL);
 807
 808         for (i = 0; i < n_fields; i++) {
 809                 struct rte_swx_field_params *f = &fields[i];
 810                 uint32_t j;
 811
 812                 CHECK_NAME(f->name, EINVAL);
 813                 CHECK(f->n_bits, EINVAL);
 814                 CHECK(f->n_bits <= 64, EINVAL);
 815                 CHECK((f->n_bits & 7) == 0, EINVAL);
 816
 817                 for (j = 0; j < i; j++) {
 818                         struct rte_swx_field_params *f_prev = &fields[j];
 819
 820                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 821                 }
 822         }
 823
 824         CHECK(!struct_type_find(p, name), EEXIST);
 825
 826         /* Node allocation. */
 827         st = calloc(1, sizeof(struct struct_type));
 828         CHECK(st, ENOMEM);
 829
 830         st->fields = calloc(n_fields, sizeof(struct field));
 831         if (!st->fields) {
 832                 free(st);
 833                 CHECK(0, ENOMEM);
 834         }
 835
 836         /* Node initialization. */
 837         strcpy(st->name, name);
 838         for (i = 0; i < n_fields; i++) {
 839                 struct field *dst = &st->fields[i];
 840                 struct rte_swx_field_params *src = &fields[i];
 841
 842                 strcpy(dst->name, src->name);
 843                 dst->n_bits = src->n_bits;
 844                 dst->offset = st->n_bits;
 845
 846                 st->n_bits += src->n_bits;
 847         }
 848         st->n_fields = n_fields;
 849
 850         /* Node add to tailq. */
 851         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 852
 853         return 0;
 854 }
 855
 856 static int
 857 struct_build(struct rte_swx_pipeline *p)
 858 {
 859         uint32_t i;
 860
 861         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 862                 struct thread *t = &p->threads[i];
 863
 864                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 865                 CHECK(t->structs, ENOMEM);
 866         }
 867
 868         return 0;
 869 }
 870
 871 static void
 872 struct_build_free(struct rte_swx_pipeline *p)
 873 {
 874         uint32_t i;
 875
 876         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 877                 struct thread *t = &p->threads[i];
 878
 879                 free(t->structs);
 880                 t->structs = NULL;
 881         }
 882 }
 883
 884 static void
 885 struct_free(struct rte_swx_pipeline *p)
 886 {
 887         struct_build_free(p);
 888
 889         /* Struct types. */
 890         for ( ; ; ) {
 891                 struct struct_type *elem;
 892
 893                 elem = TAILQ_FIRST(&p->struct_types);
 894                 if (!elem)
 895                         break;
 896
 897                 TAILQ_REMOVE(&p->struct_types, elem, node);
 898                 free(elem->fields);
 899                 free(elem);
 900         }
 901 }
 902
 903 /*
 904  * Input port.
 905  */
 906 static struct port_in_type *
 907 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 908 {
 909         struct port_in_type *elem;
 910
 911         if (!name)
 912                 return NULL;
 913
 914         TAILQ_FOREACH(elem, &p->port_in_types, node)
 915                 if (strcmp(elem->name, name) == 0)
 916                         return elem;
 917
 918         return NULL;
 919 }
 920
 921 int
 922 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 923                                        const char *name,
 924                                        struct rte_swx_port_in_ops *ops)
 925 {
 926         struct port_in_type *elem;
 927
 928         CHECK(p, EINVAL);
 929         CHECK_NAME(name, EINVAL);
 930         CHECK(ops, EINVAL);
 931         CHECK(ops->create, EINVAL);
 932         CHECK(ops->free, EINVAL);
 933         CHECK(ops->pkt_rx, EINVAL);
 934         CHECK(ops->stats_read, EINVAL);
 935
 936         CHECK(!port_in_type_find(p, name), EEXIST);
 937
 938         /* Node allocation. */
 939         elem = calloc(1, sizeof(struct port_in_type));
 940         CHECK(elem, ENOMEM);
 941
 942         /* Node initialization. */
 943         strcpy(elem->name, name);
 944         memcpy(&elem->ops, ops, sizeof(*ops));
 945
 946         /* Node add to tailq. */
 947         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 948
 949         return 0;
 950 }
 951
 952 static struct port_in *
 953 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 954 {
 955         struct port_in *port;
 956
 957         TAILQ_FOREACH(port, &p->ports_in, node)
 958                 if (port->id == port_id)
 959                         return port;
 960
 961         return NULL;
 962 }
 963
 964 int
 965 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 966                                 uint32_t port_id,
 967                                 const char *port_type_name,
 968                                 void *args)
 969 {
 970         struct port_in_type *type = NULL;
 971         struct port_in *port = NULL;
 972         void *obj = NULL;
 973
 974         CHECK(p, EINVAL);
 975
 976         CHECK(!port_in_find(p, port_id), EINVAL);
 977
 978         CHECK_NAME(port_type_name, EINVAL);
 979         type = port_in_type_find(p, port_type_name);
 980         CHECK(type, EINVAL);
 981
 982         obj = type->ops.create(args);
 983         CHECK(obj, ENODEV);
 984
 985         /* Node allocation. */
 986         port = calloc(1, sizeof(struct port_in));
 987         CHECK(port, ENOMEM);
 988
 989         /* Node initialization. */
 990         port->type = type;
 991         port->obj = obj;
 992         port->id = port_id;
 993
 994         /* Node add to tailq. */
 995         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 996         if (p->n_ports_in < port_id + 1)
 997                 p->n_ports_in = port_id + 1;
 998
 999         return 0;
1000 }
1001
1002 static int
1003 port_in_build(struct rte_swx_pipeline *p)
1004 {
1005         struct port_in *port;
1006         uint32_t i;
1007
1008         CHECK(p->n_ports_in, EINVAL);
1009         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1010
1011         for (i = 0; i < p->n_ports_in; i++)
1012                 CHECK(port_in_find(p, i), EINVAL);
1013
1014         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1015         CHECK(p->in, ENOMEM);
1016
1017         TAILQ_FOREACH(port, &p->ports_in, node) {
1018                 struct port_in_runtime *in = &p->in[port->id];
1019
1020                 in->pkt_rx = port->type->ops.pkt_rx;
1021                 in->obj = port->obj;
1022         }
1023
1024         return 0;
1025 }
1026
1027 static void
1028 port_in_build_free(struct rte_swx_pipeline *p)
1029 {
1030         free(p->in);
1031         p->in = NULL;
1032 }
1033
1034 static void
1035 port_in_free(struct rte_swx_pipeline *p)
1036 {
1037         port_in_build_free(p);
1038
1039         /* Input ports. */
1040         for ( ; ; ) {
1041                 struct port_in *port;
1042
1043                 port = TAILQ_FIRST(&p->ports_in);
1044                 if (!port)
1045                         break;
1046
1047                 TAILQ_REMOVE(&p->ports_in, port, node);
1048                 port->type->ops.free(port->obj);
1049                 free(port);
1050         }
1051
1052         /* Input port types. */
1053         for ( ; ; ) {
1054                 struct port_in_type *elem;
1055
1056                 elem = TAILQ_FIRST(&p->port_in_types);
1057                 if (!elem)
1058                         break;
1059
1060                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1061                 free(elem);
1062         }
1063 }
1064
1065 /*
1066  * Output port.
1067  */
1068 static struct port_out_type *
1069 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1070 {
1071         struct port_out_type *elem;
1072
1073         if (!name)
1074                 return NULL;
1075
1076         TAILQ_FOREACH(elem, &p->port_out_types, node)
1077                 if (!strcmp(elem->name, name))
1078                         return elem;
1079
1080         return NULL;
1081 }
1082
1083 int
1084 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1085                                         const char *name,
1086                                         struct rte_swx_port_out_ops *ops)
1087 {
1088         struct port_out_type *elem;
1089
1090         CHECK(p, EINVAL);
1091         CHECK_NAME(name, EINVAL);
1092         CHECK(ops, EINVAL);
1093         CHECK(ops->create, EINVAL);
1094         CHECK(ops->free, EINVAL);
1095         CHECK(ops->pkt_tx, EINVAL);
1096         CHECK(ops->stats_read, EINVAL);
1097
1098         CHECK(!port_out_type_find(p, name), EEXIST);
1099
1100         /* Node allocation. */
1101         elem = calloc(1, sizeof(struct port_out_type));
1102         CHECK(elem, ENOMEM);
1103
1104         /* Node initialization. */
1105         strcpy(elem->name, name);
1106         memcpy(&elem->ops, ops, sizeof(*ops));
1107
1108         /* Node add to tailq. */
1109         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1110
1111         return 0;
1112 }
1113
1114 static struct port_out *
1115 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1116 {
1117         struct port_out *port;
1118
1119         TAILQ_FOREACH(port, &p->ports_out, node)
1120                 if (port->id == port_id)
1121                         return port;
1122
1123         return NULL;
1124 }
1125
1126 int
1127 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1128                                  uint32_t port_id,
1129                                  const char *port_type_name,
1130                                  void *args)
1131 {
1132         struct port_out_type *type = NULL;
1133         struct port_out *port = NULL;
1134         void *obj = NULL;
1135
1136         CHECK(p, EINVAL);
1137
1138         CHECK(!port_out_find(p, port_id), EINVAL);
1139
1140         CHECK_NAME(port_type_name, EINVAL);
1141         type = port_out_type_find(p, port_type_name);
1142         CHECK(type, EINVAL);
1143
1144         obj = type->ops.create(args);
1145         CHECK(obj, ENODEV);
1146
1147         /* Node allocation. */
1148         port = calloc(1, sizeof(struct port_out));
1149         CHECK(port, ENOMEM);
1150
1151         /* Node initialization. */
1152         port->type = type;
1153         port->obj = obj;
1154         port->id = port_id;
1155
1156         /* Node add to tailq. */
1157         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1158         if (p->n_ports_out < port_id + 1)
1159                 p->n_ports_out = port_id + 1;
1160
1161         return 0;
1162 }
1163
1164 static int
1165 port_out_build(struct rte_swx_pipeline *p)
1166 {
1167         struct port_out *port;
1168         uint32_t i;
1169
1170         CHECK(p->n_ports_out, EINVAL);
1171
1172         for (i = 0; i < p->n_ports_out; i++)
1173                 CHECK(port_out_find(p, i), EINVAL);
1174
1175         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1176         CHECK(p->out, ENOMEM);
1177
1178         TAILQ_FOREACH(port, &p->ports_out, node) {
1179                 struct port_out_runtime *out = &p->out[port->id];
1180
1181                 out->pkt_tx = port->type->ops.pkt_tx;
1182                 out->flush = port->type->ops.flush;
1183                 out->obj = port->obj;
1184         }
1185
1186         return 0;
1187 }
1188
1189 static void
1190 port_out_build_free(struct rte_swx_pipeline *p)
1191 {
1192         free(p->out);
1193         p->out = NULL;
1194 }
1195
1196 static void
1197 port_out_free(struct rte_swx_pipeline *p)
1198 {
1199         port_out_build_free(p);
1200
1201         /* Output ports. */
1202         for ( ; ; ) {
1203                 struct port_out *port;
1204
1205                 port = TAILQ_FIRST(&p->ports_out);
1206                 if (!port)
1207                         break;
1208
1209                 TAILQ_REMOVE(&p->ports_out, port, node);
1210                 port->type->ops.free(port->obj);
1211                 free(port);
1212         }
1213
1214         /* Output port types. */
1215         for ( ; ; ) {
1216                 struct port_out_type *elem;
1217
1218                 elem = TAILQ_FIRST(&p->port_out_types);
1219                 if (!elem)
1220                         break;
1221
1222                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1223                 free(elem);
1224         }
1225 }
1226
1227 /*
1228  * Extern object.
1229  */
1230 static struct extern_type *
1231 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1232 {
1233         struct extern_type *elem;
1234
1235         TAILQ_FOREACH(elem, &p->extern_types, node)
1236                 if (strcmp(elem->name, name) == 0)
1237                         return elem;
1238
1239         return NULL;
1240 }
1241
1242 static struct extern_type_member_func *
1243 extern_type_member_func_find(struct extern_type *type, const char *name)
1244 {
1245         struct extern_type_member_func *elem;
1246
1247         TAILQ_FOREACH(elem, &type->funcs, node)
1248                 if (strcmp(elem->name, name) == 0)
1249                         return elem;
1250
1251         return NULL;
1252 }
1253
1254 static struct extern_obj *
1255 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1256 {
1257         struct extern_obj *elem;
1258
1259         TAILQ_FOREACH(elem, &p->extern_objs, node)
1260                 if (strcmp(elem->name, name) == 0)
1261                         return elem;
1262
1263         return NULL;
1264 }
1265
1266 static struct field *
1267 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1268                                const char *name,
1269                                struct extern_obj **object)
1270 {
1271         struct extern_obj *obj;
1272         struct field *f;
1273         char *obj_name, *field_name;
1274
1275         if ((name[0] != 'e') || (name[1] != '.'))
1276                 return NULL;
1277
1278         obj_name = strdup(&name[2]);
1279         if (!obj_name)
1280                 return NULL;
1281
1282         field_name = strchr(obj_name, '.');
1283         if (!field_name) {
1284                 free(obj_name);
1285                 return NULL;
1286         }
1287
1288         *field_name = 0;
1289         field_name++;
1290
1291         obj = extern_obj_find(p, obj_name);
1292         if (!obj) {
1293                 free(obj_name);
1294                 return NULL;
1295         }
1296
1297         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1298         if (!f) {
1299                 free(obj_name);
1300                 return NULL;
1301         }
1302
1303         if (object)
1304                 *object = obj;
1305
1306         free(obj_name);
1307         return f;
1308 }
1309
1310 int
1311 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1312         const char *name,
1313         const char *mailbox_struct_type_name,
1314         rte_swx_extern_type_constructor_t constructor,
1315         rte_swx_extern_type_destructor_t destructor)
1316 {
1317         struct extern_type *elem;
1318         struct struct_type *mailbox_struct_type;
1319
1320         CHECK(p, EINVAL);
1321
1322         CHECK_NAME(name, EINVAL);
1323         CHECK(!extern_type_find(p, name), EEXIST);
1324
1325         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1326         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1327         CHECK(mailbox_struct_type, EINVAL);
1328
1329         CHECK(constructor, EINVAL);
1330         CHECK(destructor, EINVAL);
1331
1332         /* Node allocation. */
1333         elem = calloc(1, sizeof(struct extern_type));
1334         CHECK(elem, ENOMEM);
1335
1336         /* Node initialization. */
1337         strcpy(elem->name, name);
1338         elem->mailbox_struct_type = mailbox_struct_type;
1339         elem->constructor = constructor;
1340         elem->destructor = destructor;
1341         TAILQ_INIT(&elem->funcs);
1342
1343         /* Node add to tailq. */
1344         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1345
1346         return 0;
1347 }
1348
1349 int
1350 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1351         const char *extern_type_name,
1352         const char *name,
1353         rte_swx_extern_type_member_func_t member_func)
1354 {
1355         struct extern_type *type;
1356         struct extern_type_member_func *type_member;
1357
1358         CHECK(p, EINVAL);
1359
1360         CHECK(extern_type_name, EINVAL);
1361         type = extern_type_find(p, extern_type_name);
1362         CHECK(type, EINVAL);
1363         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1364
1365         CHECK(name, EINVAL);
1366         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1367
1368         CHECK(member_func, EINVAL);
1369
1370         /* Node allocation. */
1371         type_member = calloc(1, sizeof(struct extern_type_member_func));
1372         CHECK(type_member, ENOMEM);
1373
1374         /* Node initialization. */
1375         strcpy(type_member->name, name);
1376         type_member->func = member_func;
1377         type_member->id = type->n_funcs;
1378
1379         /* Node add to tailq. */
1380         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1381         type->n_funcs++;
1382
1383         return 0;
1384 }
1385
1386 int
1387 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1388                                       const char *extern_type_name,
1389                                       const char *name,
1390                                       const char *args)
1391 {
1392         struct extern_type *type;
1393         struct extern_obj *obj;
1394         void *obj_handle;
1395
1396         CHECK(p, EINVAL);
1397
1398         CHECK_NAME(extern_type_name, EINVAL);
1399         type = extern_type_find(p, extern_type_name);
1400         CHECK(type, EINVAL);
1401
1402         CHECK_NAME(name, EINVAL);
1403         CHECK(!extern_obj_find(p, name), EEXIST);
1404
1405         /* Node allocation. */
1406         obj = calloc(1, sizeof(struct extern_obj));
1407         CHECK(obj, ENOMEM);
1408
1409         /* Object construction. */
1410         obj_handle = type->constructor(args);
1411         if (!obj_handle) {
1412                 free(obj);
1413                 CHECK(0, ENODEV);
1414         }
1415
1416         /* Node initialization. */
1417         strcpy(obj->name, name);
1418         obj->type = type;
1419         obj->obj = obj_handle;
1420         obj->struct_id = p->n_structs;
1421         obj->id = p->n_extern_objs;
1422
1423         /* Node add to tailq. */
1424         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1425         p->n_extern_objs++;
1426         p->n_structs++;
1427
1428         return 0;
1429 }
1430
1431 static int
1432 extern_obj_build(struct rte_swx_pipeline *p)
1433 {
1434         uint32_t i;
1435
1436         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1437                 struct thread *t = &p->threads[i];
1438                 struct extern_obj *obj;
1439
1440                 t->extern_objs = calloc(p->n_extern_objs,
1441                                         sizeof(struct extern_obj_runtime));
1442                 CHECK(t->extern_objs, ENOMEM);
1443
1444                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1445                         struct extern_obj_runtime *r =
1446                                 &t->extern_objs[obj->id];
1447                         struct extern_type_member_func *func;
1448                         uint32_t mailbox_size =
1449                                 obj->type->mailbox_struct_type->n_bits / 8;
1450
1451                         r->obj = obj->obj;
1452
1453                         r->mailbox = calloc(1, mailbox_size);
1454                         CHECK(r->mailbox, ENOMEM);
1455
1456                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1457                                 r->funcs[func->id] = func->func;
1458
1459                         t->structs[obj->struct_id] = r->mailbox;
1460                 }
1461         }
1462
1463         return 0;
1464 }
1465
1466 static void
1467 extern_obj_build_free(struct rte_swx_pipeline *p)
1468 {
1469         uint32_t i;
1470
1471         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1472                 struct thread *t = &p->threads[i];
1473                 uint32_t j;
1474
1475                 if (!t->extern_objs)
1476                         continue;
1477
1478                 for (j = 0; j < p->n_extern_objs; j++) {
1479                         struct extern_obj_runtime *r = &t->extern_objs[j];
1480
1481                         free(r->mailbox);
1482                 }
1483
1484                 free(t->extern_objs);
1485                 t->extern_objs = NULL;
1486         }
1487 }
1488
1489 static void
1490 extern_obj_free(struct rte_swx_pipeline *p)
1491 {
1492         extern_obj_build_free(p);
1493
1494         /* Extern objects. */
1495         for ( ; ; ) {
1496                 struct extern_obj *elem;
1497
1498                 elem = TAILQ_FIRST(&p->extern_objs);
1499                 if (!elem)
1500                         break;
1501
1502                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1503                 if (elem->obj)
1504                         elem->type->destructor(elem->obj);
1505                 free(elem);
1506         }
1507
1508         /* Extern types. */
1509         for ( ; ; ) {
1510                 struct extern_type *elem;
1511
1512                 elem = TAILQ_FIRST(&p->extern_types);
1513                 if (!elem)
1514                         break;
1515
1516                 TAILQ_REMOVE(&p->extern_types, elem, node);
1517
1518                 for ( ; ; ) {
1519                         struct extern_type_member_func *func;
1520
1521                         func = TAILQ_FIRST(&elem->funcs);
1522                         if (!func)
1523                                 break;
1524
1525                         TAILQ_REMOVE(&elem->funcs, func, node);
1526                         free(func);
1527                 }
1528
1529                 free(elem);
1530         }
1531 }
1532
1533 /*
1534  * Extern function.
1535  */
1536 static struct extern_func *
1537 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1538 {
1539         struct extern_func *elem;
1540
1541         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1542                 if (strcmp(elem->name, name) == 0)
1543                         return elem;
1544
1545         return NULL;
1546 }
1547
1548 static struct field *
1549 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1550                                 const char *name,
1551                                 struct extern_func **function)
1552 {
1553         struct extern_func *func;
1554         struct field *f;
1555         char *func_name, *field_name;
1556
1557         if ((name[0] != 'f') || (name[1] != '.'))
1558                 return NULL;
1559
1560         func_name = strdup(&name[2]);
1561         if (!func_name)
1562                 return NULL;
1563
1564         field_name = strchr(func_name, '.');
1565         if (!field_name) {
1566                 free(func_name);
1567                 return NULL;
1568         }
1569
1570         *field_name = 0;
1571         field_name++;
1572
1573         func = extern_func_find(p, func_name);
1574         if (!func) {
1575                 free(func_name);
1576                 return NULL;
1577         }
1578
1579         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1580         if (!f) {
1581                 free(func_name);
1582                 return NULL;
1583         }
1584
1585         if (function)
1586                 *function = func;
1587
1588         free(func_name);
1589         return f;
1590 }
1591
1592 int
1593 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1594                                       const char *name,
1595                                       const char *mailbox_struct_type_name,
1596                                       rte_swx_extern_func_t func)
1597 {
1598         struct extern_func *f;
1599         struct struct_type *mailbox_struct_type;
1600
1601         CHECK(p, EINVAL);
1602
1603         CHECK_NAME(name, EINVAL);
1604         CHECK(!extern_func_find(p, name), EEXIST);
1605
1606         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1607         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1608         CHECK(mailbox_struct_type, EINVAL);
1609
1610         CHECK(func, EINVAL);
1611
1612         /* Node allocation. */
1613         f = calloc(1, sizeof(struct extern_func));
1614         CHECK(func, ENOMEM);
1615
1616         /* Node initialization. */
1617         strcpy(f->name, name);
1618         f->mailbox_struct_type = mailbox_struct_type;
1619         f->func = func;
1620         f->struct_id = p->n_structs;
1621         f->id = p->n_extern_funcs;
1622
1623         /* Node add to tailq. */
1624         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1625         p->n_extern_funcs++;
1626         p->n_structs++;
1627
1628         return 0;
1629 }
1630
1631 static int
1632 extern_func_build(struct rte_swx_pipeline *p)
1633 {
1634         uint32_t i;
1635
1636         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1637                 struct thread *t = &p->threads[i];
1638                 struct extern_func *func;
1639
1640                 /* Memory allocation. */
1641                 t->extern_funcs = calloc(p->n_extern_funcs,
1642                                          sizeof(struct extern_func_runtime));
1643                 CHECK(t->extern_funcs, ENOMEM);
1644
1645                 /* Extern function. */
1646                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1647                         struct extern_func_runtime *r =
1648                                 &t->extern_funcs[func->id];
1649                         uint32_t mailbox_size =
1650                                 func->mailbox_struct_type->n_bits / 8;
1651
1652                         r->func = func->func;
1653
1654                         r->mailbox = calloc(1, mailbox_size);
1655                         CHECK(r->mailbox, ENOMEM);
1656
1657                         t->structs[func->struct_id] = r->mailbox;
1658                 }
1659         }
1660
1661         return 0;
1662 }
1663
1664 static void
1665 extern_func_build_free(struct rte_swx_pipeline *p)
1666 {
1667         uint32_t i;
1668
1669         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1670                 struct thread *t = &p->threads[i];
1671                 uint32_t j;
1672
1673                 if (!t->extern_funcs)
1674                         continue;
1675
1676                 for (j = 0; j < p->n_extern_funcs; j++) {
1677                         struct extern_func_runtime *r = &t->extern_funcs[j];
1678
1679                         free(r->mailbox);
1680                 }
1681
1682                 free(t->extern_funcs);
1683                 t->extern_funcs = NULL;
1684         }
1685 }
1686
1687 static void
1688 extern_func_free(struct rte_swx_pipeline *p)
1689 {
1690         extern_func_build_free(p);
1691
1692         for ( ; ; ) {
1693                 struct extern_func *elem;
1694
1695                 elem = TAILQ_FIRST(&p->extern_funcs);
1696                 if (!elem)
1697                         break;
1698
1699                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1700                 free(elem);
1701         }
1702 }
1703
1704 /*
1705  * Header.
1706  */
1707 static struct header *
1708 header_find(struct rte_swx_pipeline *p, const char *name)
1709 {
1710         struct header *elem;
1711
1712         TAILQ_FOREACH(elem, &p->headers, node)
1713                 if (strcmp(elem->name, name) == 0)
1714                         return elem;
1715
1716         return NULL;
1717 }
1718
1719 static struct header *
1720 header_parse(struct rte_swx_pipeline *p,
1721              const char *name)
1722 {
1723         if (name[0] != 'h' || name[1] != '.')
1724                 return NULL;
1725
1726         return header_find(p, &name[2]);
1727 }
1728
1729 static struct field *
1730 header_field_parse(struct rte_swx_pipeline *p,
1731                    const char *name,
1732                    struct header **header)
1733 {
1734         struct header *h;
1735         struct field *f;
1736         char *header_name, *field_name;
1737
1738         if ((name[0] != 'h') || (name[1] != '.'))
1739                 return NULL;
1740
1741         header_name = strdup(&name[2]);
1742         if (!header_name)
1743                 return NULL;
1744
1745         field_name = strchr(header_name, '.');
1746         if (!field_name) {
1747                 free(header_name);
1748                 return NULL;
1749         }
1750
1751         *field_name = 0;
1752         field_name++;
1753
1754         h = header_find(p, header_name);
1755         if (!h) {
1756                 free(header_name);
1757                 return NULL;
1758         }
1759
1760         f = struct_type_field_find(h->st, field_name);
1761         if (!f) {
1762                 free(header_name);
1763                 return NULL;
1764         }
1765
1766         if (header)
1767                 *header = h;
1768
1769         free(header_name);
1770         return f;
1771 }
1772
1773 int
1774 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1775                                         const char *name,
1776                                         const char *struct_type_name)
1777 {
1778         struct struct_type *st;
1779         struct header *h;
1780         size_t n_headers_max;
1781
1782         CHECK(p, EINVAL);
1783         CHECK_NAME(name, EINVAL);
1784         CHECK_NAME(struct_type_name, EINVAL);
1785
1786         CHECK(!header_find(p, name), EEXIST);
1787
1788         st = struct_type_find(p, struct_type_name);
1789         CHECK(st, EINVAL);
1790
1791         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1792         CHECK(p->n_headers < n_headers_max, ENOSPC);
1793
1794         /* Node allocation. */
1795         h = calloc(1, sizeof(struct header));
1796         CHECK(h, ENOMEM);
1797
1798         /* Node initialization. */
1799         strcpy(h->name, name);
1800         h->st = st;
1801         h->struct_id = p->n_structs;
1802         h->id = p->n_headers;
1803
1804         /* Node add to tailq. */
1805         TAILQ_INSERT_TAIL(&p->headers, h, node);
1806         p->n_headers++;
1807         p->n_structs++;
1808
1809         return 0;
1810 }
1811
1812 static int
1813 header_build(struct rte_swx_pipeline *p)
1814 {
1815         struct header *h;
1816         uint32_t n_bytes = 0, i;
1817
1818         TAILQ_FOREACH(h, &p->headers, node) {
1819                 n_bytes += h->st->n_bits / 8;
1820         }
1821
1822         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1823                 struct thread *t = &p->threads[i];
1824                 uint32_t offset = 0;
1825
1826                 t->headers = calloc(p->n_headers,
1827                                     sizeof(struct header_runtime));
1828                 CHECK(t->headers, ENOMEM);
1829
1830                 t->headers_out = calloc(p->n_headers,
1831                                         sizeof(struct header_out_runtime));
1832                 CHECK(t->headers_out, ENOMEM);
1833
1834                 t->header_storage = calloc(1, n_bytes);
1835                 CHECK(t->header_storage, ENOMEM);
1836
1837                 t->header_out_storage = calloc(1, n_bytes);
1838                 CHECK(t->header_out_storage, ENOMEM);
1839
1840                 TAILQ_FOREACH(h, &p->headers, node) {
1841                         uint8_t *header_storage;
1842
1843                         header_storage = &t->header_storage[offset];
1844                         offset += h->st->n_bits / 8;
1845
1846                         t->headers[h->id].ptr0 = header_storage;
1847                         t->structs[h->struct_id] = header_storage;
1848                 }
1849         }
1850
1851         return 0;
1852 }
1853
1854 static void
1855 header_build_free(struct rte_swx_pipeline *p)
1856 {
1857         uint32_t i;
1858
1859         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1860                 struct thread *t = &p->threads[i];
1861
1862                 free(t->headers_out);
1863                 t->headers_out = NULL;
1864
1865                 free(t->headers);
1866                 t->headers = NULL;
1867
1868                 free(t->header_out_storage);
1869                 t->header_out_storage = NULL;
1870
1871                 free(t->header_storage);
1872                 t->header_storage = NULL;
1873         }
1874 }
1875
1876 static void
1877 header_free(struct rte_swx_pipeline *p)
1878 {
1879         header_build_free(p);
1880
1881         for ( ; ; ) {
1882                 struct header *elem;
1883
1884                 elem = TAILQ_FIRST(&p->headers);
1885                 if (!elem)
1886                         break;
1887
1888                 TAILQ_REMOVE(&p->headers, elem, node);
1889                 free(elem);
1890         }
1891 }
1892
1893 /*
1894  * Meta-data.
1895  */
1896 static struct field *
1897 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1898 {
1899         if (!p->metadata_st)
1900                 return NULL;
1901
1902         if (name[0] != 'm' || name[1] != '.')
1903                 return NULL;
1904
1905         return struct_type_field_find(p->metadata_st, &name[2]);
1906 }
1907
1908 int
1909 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1910                                           const char *struct_type_name)
1911 {
1912         struct struct_type *st = NULL;
1913
1914         CHECK(p, EINVAL);
1915
1916         CHECK_NAME(struct_type_name, EINVAL);
1917         st  = struct_type_find(p, struct_type_name);
1918         CHECK(st, EINVAL);
1919         CHECK(!p->metadata_st, EINVAL);
1920
1921         p->metadata_st = st;
1922         p->metadata_struct_id = p->n_structs;
1923
1924         p->n_structs++;
1925
1926         return 0;
1927 }
1928
1929 static int
1930 metadata_build(struct rte_swx_pipeline *p)
1931 {
1932         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1933         uint32_t i;
1934
1935         /* Thread-level initialization. */
1936         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1937                 struct thread *t = &p->threads[i];
1938                 uint8_t *metadata;
1939
1940                 metadata = calloc(1, n_bytes);
1941                 CHECK(metadata, ENOMEM);
1942
1943                 t->metadata = metadata;
1944                 t->structs[p->metadata_struct_id] = metadata;
1945         }
1946
1947         return 0;
1948 }
1949
1950 static void
1951 metadata_build_free(struct rte_swx_pipeline *p)
1952 {
1953         uint32_t i;
1954
1955         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1956                 struct thread *t = &p->threads[i];
1957
1958                 free(t->metadata);
1959                 t->metadata = NULL;
1960         }
1961 }
1962
1963 static void
1964 metadata_free(struct rte_swx_pipeline *p)
1965 {
1966         metadata_build_free(p);
1967 }
1968
1969 /*
1970  * Instruction.
1971  */
1972 static struct field *
1973 action_field_parse(struct action *action, const char *name);
1974
1975 static struct field *
1976 struct_field_parse(struct rte_swx_pipeline *p,
1977                    struct action *action,
1978                    const char *name,
1979                    uint32_t *struct_id)
1980 {
1981         struct field *f;
1982
1983         switch (name[0]) {
1984         case 'h':
1985         {
1986                 struct header *header;
1987
1988                 f = header_field_parse(p, name, &header);
1989                 if (!f)
1990                         return NULL;
1991
1992                 *struct_id = header->struct_id;
1993                 return f;
1994         }
1995
1996         case 'm':
1997         {
1998                 f = metadata_field_parse(p, name);
1999                 if (!f)
2000                         return NULL;
2001
2002                 *struct_id = p->metadata_struct_id;
2003                 return f;
2004         }
2005
2006         case 't':
2007         {
2008                 if (!action)
2009                         return NULL;
2010
2011                 f = action_field_parse(action, name);
2012                 if (!f)
2013                         return NULL;
2014
2015                 *struct_id = 0;
2016                 return f;
2017         }
2018
2019         case 'e':
2020         {
2021                 struct extern_obj *obj;
2022
2023                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2024                 if (!f)
2025                         return NULL;
2026
2027                 *struct_id = obj->struct_id;
2028                 return f;
2029         }
2030
2031         case 'f':
2032         {
2033                 struct extern_func *func;
2034
2035                 f = extern_func_mailbox_field_parse(p, name, &func);
2036                 if (!f)
2037                         return NULL;
2038
2039                 *struct_id = func->struct_id;
2040                 return f;
2041         }
2042
2043         default:
2044                 return NULL;
2045         }
2046 }
2047
2048 static inline void
2049 pipeline_port_inc(struct rte_swx_pipeline *p)
2050 {
2051         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2052 }
2053
2054 static inline void
2055 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2056 {
2057         t->ip = p->instructions;
2058 }
2059
2060 static inline void
2061 thread_ip_inc(struct rte_swx_pipeline *p);
2062
2063 static inline void
2064 thread_ip_inc(struct rte_swx_pipeline *p)
2065 {
2066         struct thread *t = &p->threads[p->thread_id];
2067
2068         t->ip++;
2069 }
2070
2071 static inline void
2072 thread_ip_inc_cond(struct thread *t, int cond)
2073 {
2074         t->ip += cond;
2075 }
2076
2077 static inline void
2078 thread_yield(struct rte_swx_pipeline *p)
2079 {
2080         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2081 }
2082
2083 /*
2084  * rx.
2085  */
2086 static int
2087 instr_rx_translate(struct rte_swx_pipeline *p,
2088                    struct action *action,
2089                    char **tokens,
2090                    int n_tokens,
2091                    struct instruction *instr,
2092                    struct instruction_data *data __rte_unused)
2093 {
2094         struct field *f;
2095
2096         CHECK(!action, EINVAL);
2097         CHECK(n_tokens == 2, EINVAL);
2098
2099         f = metadata_field_parse(p, tokens[1]);
2100         CHECK(f, EINVAL);
2101
2102         instr->type = INSTR_RX;
2103         instr->io.io.offset = f->offset / 8;
2104         instr->io.io.n_bits = f->n_bits;
2105         return 0;
2106 }
2107
2108 static inline void
2109 instr_rx_exec(struct rte_swx_pipeline *p);
2110
2111 static inline void
2112 instr_rx_exec(struct rte_swx_pipeline *p)
2113 {
2114         struct thread *t = &p->threads[p->thread_id];
2115         struct instruction *ip = t->ip;
2116         struct port_in_runtime *port = &p->in[p->port_id];
2117         struct rte_swx_pkt *pkt = &t->pkt;
2118         int pkt_received;
2119
2120         /* Packet. */
2121         pkt_received = port->pkt_rx(port->obj, pkt);
2122         t->ptr = &pkt->pkt[pkt->offset];
2123         rte_prefetch0(t->ptr);
2124
2125         TRACE("[Thread %2u] rx %s from port %u\n",
2126               p->thread_id,
2127               pkt_received ? "1 pkt" : "0 pkts",
2128               p->port_id);
2129
2130         /* Headers. */
2131         t->valid_headers = 0;
2132         t->n_headers_out = 0;
2133
2134         /* Meta-data. */
2135         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2136
2137         /* Tables. */
2138         t->table_state = p->table_state;
2139
2140         /* Thread. */
2141         pipeline_port_inc(p);
2142         thread_ip_inc_cond(t, pkt_received);
2143         thread_yield(p);
2144 }
2145
2146 /*
2147  * tx.
2148  */
2149 static int
2150 instr_tx_translate(struct rte_swx_pipeline *p,
2151                    struct action *action __rte_unused,
2152                    char **tokens,
2153                    int n_tokens,
2154                    struct instruction *instr,
2155                    struct instruction_data *data __rte_unused)
2156 {
2157         struct field *f;
2158
2159         CHECK(n_tokens == 2, EINVAL);
2160
2161         f = metadata_field_parse(p, tokens[1]);
2162         CHECK(f, EINVAL);
2163
2164         instr->type = INSTR_TX;
2165         instr->io.io.offset = f->offset / 8;
2166         instr->io.io.n_bits = f->n_bits;
2167         return 0;
2168 }
2169
2170 static inline void
2171 emit_handler(struct thread *t)
2172 {
2173         struct header_out_runtime *h0 = &t->headers_out[0];
2174         struct header_out_runtime *h1 = &t->headers_out[1];
2175         uint32_t offset = 0, i;
2176
2177         /* No header change or header decapsulation. */
2178         if ((t->n_headers_out == 1) &&
2179             (h0->ptr + h0->n_bytes == t->ptr)) {
2180                 TRACE("Emit handler: no header change or header decap.\n");
2181
2182                 t->pkt.offset -= h0->n_bytes;
2183                 t->pkt.length += h0->n_bytes;
2184
2185                 return;
2186         }
2187
2188         /* Header encapsulation (optionally, with prior header decasulation). */
2189         if ((t->n_headers_out == 2) &&
2190             (h1->ptr + h1->n_bytes == t->ptr) &&
2191             (h0->ptr == h0->ptr0)) {
2192                 uint32_t offset;
2193
2194                 TRACE("Emit handler: header encapsulation.\n");
2195
2196                 offset = h0->n_bytes + h1->n_bytes;
2197                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2198                 t->pkt.offset -= offset;
2199                 t->pkt.length += offset;
2200
2201                 return;
2202         }
2203
2204         /* Header insertion. */
2205         /* TBD */
2206
2207         /* Header extraction. */
2208         /* TBD */
2209
2210         /* For any other case. */
2211         TRACE("Emit handler: complex case.\n");
2212
2213         for (i = 0; i < t->n_headers_out; i++) {
2214                 struct header_out_runtime *h = &t->headers_out[i];
2215
2216                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2217                 offset += h->n_bytes;
2218         }
2219
2220         if (offset) {
2221                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2222                 t->pkt.offset -= offset;
2223                 t->pkt.length += offset;
2224         }
2225 }
2226
2227 static inline void
2228 instr_tx_exec(struct rte_swx_pipeline *p);
2229
2230 static inline void
2231 instr_tx_exec(struct rte_swx_pipeline *p)
2232 {
2233         struct thread *t = &p->threads[p->thread_id];
2234         struct instruction *ip = t->ip;
2235         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2236         struct port_out_runtime *port = &p->out[port_id];
2237         struct rte_swx_pkt *pkt = &t->pkt;
2238
2239         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2240               p->thread_id,
2241               (uint32_t)port_id);
2242
2243         /* Headers. */
2244         emit_handler(t);
2245
2246         /* Packet. */
2247         port->pkt_tx(port->obj, pkt);
2248
2249         /* Thread. */
2250         thread_ip_reset(p, t);
2251         instr_rx_exec(p);
2252 }
2253
2254 /*
2255  * extract.
2256  */
2257 static int
2258 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2259                             struct action *action,
2260                             char **tokens,
2261                             int n_tokens,
2262                             struct instruction *instr,
2263                             struct instruction_data *data __rte_unused)
2264 {
2265         struct header *h;
2266
2267         CHECK(!action, EINVAL);
2268         CHECK(n_tokens == 2, EINVAL);
2269
2270         h = header_parse(p, tokens[1]);
2271         CHECK(h, EINVAL);
2272
2273         instr->type = INSTR_HDR_EXTRACT;
2274         instr->io.hdr.header_id[0] = h->id;
2275         instr->io.hdr.struct_id[0] = h->struct_id;
2276         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2277         return 0;
2278 }
2279
2280 static inline void
2281 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2282
2283 static inline void
2284 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2285 {
2286         struct thread *t = &p->threads[p->thread_id];
2287         struct instruction *ip = t->ip;
2288         uint64_t valid_headers = t->valid_headers;
2289         uint8_t *ptr = t->ptr;
2290         uint32_t offset = t->pkt.offset;
2291         uint32_t length = t->pkt.length;
2292         uint32_t i;
2293
2294         for (i = 0; i < n_extract; i++) {
2295                 uint32_t header_id = ip->io.hdr.header_id[i];
2296                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2297                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2298
2299                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2300                       p->thread_id,
2301                       header_id,
2302                       n_bytes);
2303
2304                 /* Headers. */
2305                 t->structs[struct_id] = ptr;
2306                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2307
2308                 /* Packet. */
2309                 offset += n_bytes;
2310                 length -= n_bytes;
2311                 ptr += n_bytes;
2312         }
2313
2314         /* Headers. */
2315         t->valid_headers = valid_headers;
2316
2317         /* Packet. */
2318         t->pkt.offset = offset;
2319         t->pkt.length = length;
2320         t->ptr = ptr;
2321 }
2322
2323 static inline void
2324 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2325 {
2326         __instr_hdr_extract_exec(p, 1);
2327
2328         /* Thread. */
2329         thread_ip_inc(p);
2330 }
2331
2332 static inline void
2333 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2334 {
2335         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2336               p->thread_id);
2337
2338         __instr_hdr_extract_exec(p, 2);
2339
2340         /* Thread. */
2341         thread_ip_inc(p);
2342 }
2343
2344 static inline void
2345 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2346 {
2347         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2348               p->thread_id);
2349
2350         __instr_hdr_extract_exec(p, 3);
2351
2352         /* Thread. */
2353         thread_ip_inc(p);
2354 }
2355
2356 static inline void
2357 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2358 {
2359         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2360               p->thread_id);
2361
2362         __instr_hdr_extract_exec(p, 4);
2363
2364         /* Thread. */
2365         thread_ip_inc(p);
2366 }
2367
2368 static inline void
2369 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2370 {
2371         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2372               p->thread_id);
2373
2374         __instr_hdr_extract_exec(p, 5);
2375
2376         /* Thread. */
2377         thread_ip_inc(p);
2378 }
2379
2380 static inline void
2381 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2382 {
2383         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2384               p->thread_id);
2385
2386         __instr_hdr_extract_exec(p, 6);
2387
2388         /* Thread. */
2389         thread_ip_inc(p);
2390 }
2391
2392 static inline void
2393 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2394 {
2395         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2396               p->thread_id);
2397
2398         __instr_hdr_extract_exec(p, 7);
2399
2400         /* Thread. */
2401         thread_ip_inc(p);
2402 }
2403
2404 static inline void
2405 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2406 {
2407         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2408               p->thread_id);
2409
2410         __instr_hdr_extract_exec(p, 8);
2411
2412         /* Thread. */
2413         thread_ip_inc(p);
2414 }
2415
2416 /*
2417  * emit.
2418  */
2419 static int
2420 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2421                          struct action *action __rte_unused,
2422                          char **tokens,
2423                          int n_tokens,
2424                          struct instruction *instr,
2425                          struct instruction_data *data __rte_unused)
2426 {
2427         struct header *h;
2428
2429         CHECK(n_tokens == 2, EINVAL);
2430
2431         h = header_parse(p, tokens[1]);
2432         CHECK(h, EINVAL);
2433
2434         instr->type = INSTR_HDR_EMIT;
2435         instr->io.hdr.header_id[0] = h->id;
2436         instr->io.hdr.struct_id[0] = h->struct_id;
2437         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2438         return 0;
2439 }
2440
2441 static inline void
2442 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
2443
2444 static inline void
2445 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
2446 {
2447         struct thread *t = &p->threads[p->thread_id];
2448         struct instruction *ip = t->ip;
2449         uint32_t n_headers_out = t->n_headers_out;
2450         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
2451         uint8_t *ho_ptr = NULL;
2452         uint32_t ho_nbytes = 0, i;
2453
2454         for (i = 0; i < n_emit; i++) {
2455                 uint32_t header_id = ip->io.hdr.header_id[i];
2456                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2457                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2458
2459                 struct header_runtime *hi = &t->headers[header_id];
2460                 uint8_t *hi_ptr = t->structs[struct_id];
2461
2462                 TRACE("[Thread %2u]: emit header %u\n",
2463                       p->thread_id,
2464                       header_id);
2465
2466                 /* Headers. */
2467                 if (!i) {
2468                         if (!t->n_headers_out) {
2469                                 ho = &t->headers_out[0];
2470
2471                                 ho->ptr0 = hi->ptr0;
2472                                 ho->ptr = hi_ptr;
2473
2474                                 ho_ptr = hi_ptr;
2475                                 ho_nbytes = n_bytes;
2476
2477                                 n_headers_out = 1;
2478
2479                                 continue;
2480                         } else {
2481                                 ho_ptr = ho->ptr;
2482                                 ho_nbytes = ho->n_bytes;
2483                         }
2484                 }
2485
2486                 if (ho_ptr + ho_nbytes == hi_ptr) {
2487                         ho_nbytes += n_bytes;
2488                 } else {
2489                         ho->n_bytes = ho_nbytes;
2490
2491                         ho++;
2492                         ho->ptr0 = hi->ptr0;
2493                         ho->ptr = hi_ptr;
2494
2495                         ho_ptr = hi_ptr;
2496                         ho_nbytes = n_bytes;
2497
2498                         n_headers_out++;
2499                 }
2500         }
2501
2502         ho->n_bytes = ho_nbytes;
2503         t->n_headers_out = n_headers_out;
2504 }
2505
2506 static inline void
2507 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2508 {
2509         __instr_hdr_emit_exec(p, 1);
2510
2511         /* Thread. */
2512         thread_ip_inc(p);
2513 }
2514
2515 static inline void
2516 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2517 {
2518         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2519               p->thread_id);
2520
2521         __instr_hdr_emit_exec(p, 1);
2522         instr_tx_exec(p);
2523 }
2524
2525 static inline void
2526 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2527 {
2528         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2529               p->thread_id);
2530
2531         __instr_hdr_emit_exec(p, 2);
2532         instr_tx_exec(p);
2533 }
2534
2535 static inline void
2536 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2537 {
2538         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2539               p->thread_id);
2540
2541         __instr_hdr_emit_exec(p, 3);
2542         instr_tx_exec(p);
2543 }
2544
2545 static inline void
2546 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2547 {
2548         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2549               p->thread_id);
2550
2551         __instr_hdr_emit_exec(p, 4);
2552         instr_tx_exec(p);
2553 }
2554
2555 static inline void
2556 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2557 {
2558         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2559               p->thread_id);
2560
2561         __instr_hdr_emit_exec(p, 5);
2562         instr_tx_exec(p);
2563 }
2564
2565 static inline void
2566 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2567 {
2568         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2569               p->thread_id);
2570
2571         __instr_hdr_emit_exec(p, 6);
2572         instr_tx_exec(p);
2573 }
2574
2575 static inline void
2576 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2577 {
2578         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2579               p->thread_id);
2580
2581         __instr_hdr_emit_exec(p, 7);
2582         instr_tx_exec(p);
2583 }
2584
2585 static inline void
2586 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2587 {
2588         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2589               p->thread_id);
2590
2591         __instr_hdr_emit_exec(p, 8);
2592         instr_tx_exec(p);
2593 }
2594
2595 /*
2596  * validate.
2597  */
2598 static int
2599 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2600                              struct action *action __rte_unused,
2601                              char **tokens,
2602                              int n_tokens,
2603                              struct instruction *instr,
2604                              struct instruction_data *data __rte_unused)
2605 {
2606         struct header *h;
2607
2608         CHECK(n_tokens == 2, EINVAL);
2609
2610         h = header_parse(p, tokens[1]);
2611         CHECK(h, EINVAL);
2612
2613         instr->type = INSTR_HDR_VALIDATE;
2614         instr->valid.header_id = h->id;
2615         return 0;
2616 }
2617
2618 static inline void
2619 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2620 {
2621         struct thread *t = &p->threads[p->thread_id];
2622         struct instruction *ip = t->ip;
2623         uint32_t header_id = ip->valid.header_id;
2624
2625         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2626
2627         /* Headers. */
2628         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2629
2630         /* Thread. */
2631         thread_ip_inc(p);
2632 }
2633
2634 /*
2635  * invalidate.
2636  */
2637 static int
2638 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2639                                struct action *action __rte_unused,
2640                                char **tokens,
2641                                int n_tokens,
2642                                struct instruction *instr,
2643                                struct instruction_data *data __rte_unused)
2644 {
2645         struct header *h;
2646
2647         CHECK(n_tokens == 2, EINVAL);
2648
2649         h = header_parse(p, tokens[1]);
2650         CHECK(h, EINVAL);
2651
2652         instr->type = INSTR_HDR_INVALIDATE;
2653         instr->valid.header_id = h->id;
2654         return 0;
2655 }
2656
2657 static inline void
2658 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2659 {
2660         struct thread *t = &p->threads[p->thread_id];
2661         struct instruction *ip = t->ip;
2662         uint32_t header_id = ip->valid.header_id;
2663
2664         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2665
2666         /* Headers. */
2667         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2668
2669         /* Thread. */
2670         thread_ip_inc(p);
2671 }
2672
2673 /*
2674  * mov.
2675  */
2676 static int
2677 instr_mov_translate(struct rte_swx_pipeline *p,
2678                     struct action *action,
2679                     char **tokens,
2680                     int n_tokens,
2681                     struct instruction *instr,
2682                     struct instruction_data *data __rte_unused)
2683 {
2684         char *dst = tokens[1], *src = tokens[2];
2685         struct field *fdst, *fsrc;
2686         uint32_t dst_struct_id, src_struct_id, src_val;
2687
2688         CHECK(n_tokens == 3, EINVAL);
2689
2690         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2691         CHECK(fdst, EINVAL);
2692
2693         /* MOV or MOV_S. */
2694         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2695         if (fsrc) {
2696                 instr->type = INSTR_MOV;
2697                 if ((dst[0] == 'h' && src[0] != 'h') ||
2698                     (dst[0] != 'h' && src[0] == 'h'))
2699                         instr->type = INSTR_MOV_S;
2700
2701                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2702                 instr->mov.dst.n_bits = fdst->n_bits;
2703                 instr->mov.dst.offset = fdst->offset / 8;
2704                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2705                 instr->mov.src.n_bits = fsrc->n_bits;
2706                 instr->mov.src.offset = fsrc->offset / 8;
2707                 return 0;
2708         }
2709
2710         /* MOV_I. */
2711         src_val = strtoul(src, &src, 0);
2712         CHECK(!src[0], EINVAL);
2713
2714         if (dst[0] == 'h')
2715                 src_val = htonl(src_val);
2716
2717         instr->type = INSTR_MOV_I;
2718         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2719         instr->mov.dst.n_bits = fdst->n_bits;
2720         instr->mov.dst.offset = fdst->offset / 8;
2721         instr->mov.src_val = (uint32_t)src_val;
2722         return 0;
2723 }
2724
2725 static inline void
2726 instr_mov_exec(struct rte_swx_pipeline *p)
2727 {
2728         struct thread *t = &p->threads[p->thread_id];
2729         struct instruction *ip = t->ip;
2730
2731         TRACE("[Thread %2u] mov\n",
2732               p->thread_id);
2733
2734         MOV(t, ip);
2735
2736         /* Thread. */
2737         thread_ip_inc(p);
2738 }
2739
2740 static inline void
2741 instr_mov_s_exec(struct rte_swx_pipeline *p)
2742 {
2743         struct thread *t = &p->threads[p->thread_id];
2744         struct instruction *ip = t->ip;
2745
2746         TRACE("[Thread %2u] mov (s)\n",
2747               p->thread_id);
2748
2749         MOV_S(t, ip);
2750
2751         /* Thread. */
2752         thread_ip_inc(p);
2753 }
2754
2755 static inline void
2756 instr_mov_i_exec(struct rte_swx_pipeline *p)
2757 {
2758         struct thread *t = &p->threads[p->thread_id];
2759         struct instruction *ip = t->ip;
2760
2761         TRACE("[Thread %2u] mov m.f %x\n",
2762               p->thread_id,
2763               ip->mov.src_val);
2764
2765         MOV_I(t, ip);
2766
2767         /* Thread. */
2768         thread_ip_inc(p);
2769 }
2770
2771 /*
2772  * dma.
2773  */
2774 static int
2775 instr_dma_translate(struct rte_swx_pipeline *p,
2776                     struct action *action,
2777                     char **tokens,
2778                     int n_tokens,
2779                     struct instruction *instr,
2780                     struct instruction_data *data __rte_unused)
2781 {
2782         char *dst = tokens[1];
2783         char *src = tokens[2];
2784         struct header *h;
2785         struct field *tf;
2786
2787         CHECK(action, EINVAL);
2788         CHECK(n_tokens == 3, EINVAL);
2789
2790         h = header_parse(p, dst);
2791         CHECK(h, EINVAL);
2792
2793         tf = action_field_parse(action, src);
2794         CHECK(tf, EINVAL);
2795
2796         instr->type = INSTR_DMA_HT;
2797         instr->dma.dst.header_id[0] = h->id;
2798         instr->dma.dst.struct_id[0] = h->struct_id;
2799         instr->dma.n_bytes[0] = h->st->n_bits / 8;
2800         instr->dma.src.offset[0] = tf->offset / 8;
2801
2802         return 0;
2803 }
2804
2805 static inline void
2806 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2807
2808 static inline void
2809 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2810 {
2811         struct thread *t = &p->threads[p->thread_id];
2812         struct instruction *ip = t->ip;
2813         uint8_t *action_data = t->structs[0];
2814         uint64_t valid_headers = t->valid_headers;
2815         uint32_t i;
2816
2817         for (i = 0; i < n_dma; i++) {
2818                 uint32_t header_id = ip->dma.dst.header_id[i];
2819                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2820                 uint32_t offset = ip->dma.src.offset[i];
2821                 uint32_t n_bytes = ip->dma.n_bytes[i];
2822
2823                 struct header_runtime *h = &t->headers[header_id];
2824                 uint8_t *h_ptr0 = h->ptr0;
2825                 uint8_t *h_ptr = t->structs[struct_id];
2826
2827                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2828                         h_ptr : h_ptr0;
2829                 void *src = &action_data[offset];
2830
2831                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2832
2833                 /* Headers. */
2834                 memcpy(dst, src, n_bytes);
2835                 t->structs[struct_id] = dst;
2836                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2837         }
2838
2839         t->valid_headers = valid_headers;
2840 }
2841
2842 static inline void
2843 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2844 {
2845         __instr_dma_ht_exec(p, 1);
2846
2847         /* Thread. */
2848         thread_ip_inc(p);
2849 }
2850
2851 static inline void
2852 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2853 {
2854         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2855               p->thread_id);
2856
2857         __instr_dma_ht_exec(p, 2);
2858
2859         /* Thread. */
2860         thread_ip_inc(p);
2861 }
2862
2863 static inline void
2864 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2865 {
2866         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2867               p->thread_id);
2868
2869         __instr_dma_ht_exec(p, 3);
2870
2871         /* Thread. */
2872         thread_ip_inc(p);
2873 }
2874
2875 static inline void
2876 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2877 {
2878         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2879               p->thread_id);
2880
2881         __instr_dma_ht_exec(p, 4);
2882
2883         /* Thread. */
2884         thread_ip_inc(p);
2885 }
2886
2887 static inline void
2888 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2889 {
2890         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2891               p->thread_id);
2892
2893         __instr_dma_ht_exec(p, 5);
2894
2895         /* Thread. */
2896         thread_ip_inc(p);
2897 }
2898
2899 static inline void
2900 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2901 {
2902         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2903               p->thread_id);
2904
2905         __instr_dma_ht_exec(p, 6);
2906
2907         /* Thread. */
2908         thread_ip_inc(p);
2909 }
2910
2911 static inline void
2912 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2913 {
2914         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2915               p->thread_id);
2916
2917         __instr_dma_ht_exec(p, 7);
2918
2919         /* Thread. */
2920         thread_ip_inc(p);
2921 }
2922
2923 static inline void
2924 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2925 {
2926         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2927               p->thread_id);
2928
2929         __instr_dma_ht_exec(p, 8);
2930
2931         /* Thread. */
2932         thread_ip_inc(p);
2933 }
2934
2935 /*
2936  * alu.
2937  */
2938 static int
2939 instr_alu_add_translate(struct rte_swx_pipeline *p,
2940                         struct action *action,
2941                         char **tokens,
2942                         int n_tokens,
2943                         struct instruction *instr,
2944                         struct instruction_data *data __rte_unused)
2945 {
2946         char *dst = tokens[1], *src = tokens[2];
2947         struct field *fdst, *fsrc;
2948         uint32_t dst_struct_id, src_struct_id, src_val;
2949
2950         CHECK(n_tokens == 3, EINVAL);
2951
2952         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2953         CHECK(fdst, EINVAL);
2954
2955         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2956         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2957         if (fsrc) {
2958                 instr->type = INSTR_ALU_ADD;
2959                 if (dst[0] == 'h' && src[0] == 'm')
2960                         instr->type = INSTR_ALU_ADD_HM;
2961                 if (dst[0] == 'm' && src[0] == 'h')
2962                         instr->type = INSTR_ALU_ADD_MH;
2963                 if (dst[0] == 'h' && src[0] == 'h')
2964                         instr->type = INSTR_ALU_ADD_HH;
2965
2966                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2967                 instr->alu.dst.n_bits = fdst->n_bits;
2968                 instr->alu.dst.offset = fdst->offset / 8;
2969                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2970                 instr->alu.src.n_bits = fsrc->n_bits;
2971                 instr->alu.src.offset = fsrc->offset / 8;
2972                 return 0;
2973         }
2974
2975         /* ADD_MI, ADD_HI. */
2976         src_val = strtoul(src, &src, 0);
2977         CHECK(!src[0], EINVAL);
2978
2979         instr->type = INSTR_ALU_ADD_MI;
2980         if (dst[0] == 'h')
2981                 instr->type = INSTR_ALU_ADD_HI;
2982
2983         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2984         instr->alu.dst.n_bits = fdst->n_bits;
2985         instr->alu.dst.offset = fdst->offset / 8;
2986         instr->alu.src_val = (uint32_t)src_val;
2987         return 0;
2988 }
2989
2990 static int
2991 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2992                         struct action *action,
2993                         char **tokens,
2994                         int n_tokens,
2995                         struct instruction *instr,
2996                         struct instruction_data *data __rte_unused)
2997 {
2998         char *dst = tokens[1], *src = tokens[2];
2999         struct field *fdst, *fsrc;
3000         uint32_t dst_struct_id, src_struct_id, src_val;
3001
3002         CHECK(n_tokens == 3, EINVAL);
3003
3004         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3005         CHECK(fdst, EINVAL);
3006
3007         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3008         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3009         if (fsrc) {
3010                 instr->type = INSTR_ALU_SUB;
3011                 if (dst[0] == 'h' && src[0] == 'm')
3012                         instr->type = INSTR_ALU_SUB_HM;
3013                 if (dst[0] == 'm' && src[0] == 'h')
3014                         instr->type = INSTR_ALU_SUB_MH;
3015                 if (dst[0] == 'h' && src[0] == 'h')
3016                         instr->type = INSTR_ALU_SUB_HH;
3017
3018                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3019                 instr->alu.dst.n_bits = fdst->n_bits;
3020                 instr->alu.dst.offset = fdst->offset / 8;
3021                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3022                 instr->alu.src.n_bits = fsrc->n_bits;
3023                 instr->alu.src.offset = fsrc->offset / 8;
3024                 return 0;
3025         }
3026
3027         /* SUB_MI, SUB_HI. */
3028         src_val = strtoul(src, &src, 0);
3029         CHECK(!src[0], EINVAL);
3030
3031         instr->type = INSTR_ALU_SUB_MI;
3032         if (dst[0] == 'h')
3033                 instr->type = INSTR_ALU_SUB_HI;
3034
3035         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3036         instr->alu.dst.n_bits = fdst->n_bits;
3037         instr->alu.dst.offset = fdst->offset / 8;
3038         instr->alu.src_val = (uint32_t)src_val;
3039         return 0;
3040 }
3041
3042 static int
3043 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3044                           struct action *action __rte_unused,
3045                           char **tokens,
3046                           int n_tokens,
3047                           struct instruction *instr,
3048                           struct instruction_data *data __rte_unused)
3049 {
3050         char *dst = tokens[1], *src = tokens[2];
3051         struct header *hdst, *hsrc;
3052         struct field *fdst, *fsrc;
3053
3054         CHECK(n_tokens == 3, EINVAL);
3055
3056         fdst = header_field_parse(p, dst, &hdst);
3057         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3058
3059         /* CKADD_FIELD. */
3060         fsrc = header_field_parse(p, src, &hsrc);
3061         if (fsrc) {
3062                 instr->type = INSTR_ALU_CKADD_FIELD;
3063                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3064                 instr->alu.dst.n_bits = fdst->n_bits;
3065                 instr->alu.dst.offset = fdst->offset / 8;
3066                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3067                 instr->alu.src.n_bits = fsrc->n_bits;
3068                 instr->alu.src.offset = fsrc->offset / 8;
3069                 return 0;
3070         }
3071
3072         /* CKADD_STRUCT, CKADD_STRUCT20. */
3073         hsrc = header_parse(p, src);
3074         CHECK(hsrc, EINVAL);
3075
3076         instr->type = INSTR_ALU_CKADD_STRUCT;
3077         if ((hsrc->st->n_bits / 8) == 20)
3078                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3079
3080         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3081         instr->alu.dst.n_bits = fdst->n_bits;
3082         instr->alu.dst.offset = fdst->offset / 8;
3083         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3084         instr->alu.src.n_bits = hsrc->st->n_bits;
3085         instr->alu.src.offset = 0; /* Unused. */
3086         return 0;
3087 }
3088
3089 static int
3090 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3091                           struct action *action __rte_unused,
3092                           char **tokens,
3093                           int n_tokens,
3094                           struct instruction *instr,
3095                           struct instruction_data *data __rte_unused)
3096 {
3097         char *dst = tokens[1], *src = tokens[2];
3098         struct header *hdst, *hsrc;
3099         struct field *fdst, *fsrc;
3100
3101         CHECK(n_tokens == 3, EINVAL);
3102
3103         fdst = header_field_parse(p, dst, &hdst);
3104         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3105
3106         fsrc = header_field_parse(p, src, &hsrc);
3107         CHECK(fsrc, EINVAL);
3108
3109         instr->type = INSTR_ALU_CKSUB_FIELD;
3110         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3111         instr->alu.dst.n_bits = fdst->n_bits;
3112         instr->alu.dst.offset = fdst->offset / 8;
3113         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3114         instr->alu.src.n_bits = fsrc->n_bits;
3115         instr->alu.src.offset = fsrc->offset / 8;
3116         return 0;
3117 }
3118
3119 static int
3120 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3121                         struct action *action,
3122                         char **tokens,
3123                         int n_tokens,
3124                         struct instruction *instr,
3125                         struct instruction_data *data __rte_unused)
3126 {
3127         char *dst = tokens[1], *src = tokens[2];
3128         struct field *fdst, *fsrc;
3129         uint32_t dst_struct_id, src_struct_id, src_val;
3130
3131         CHECK(n_tokens == 3, EINVAL);
3132
3133         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3134         CHECK(fdst, EINVAL);
3135
3136         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3137         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3138         if (fsrc) {
3139                 instr->type = INSTR_ALU_SHL;
3140                 if (dst[0] == 'h' && src[0] == 'm')
3141                         instr->type = INSTR_ALU_SHL_HM;
3142                 if (dst[0] == 'm' && src[0] == 'h')
3143                         instr->type = INSTR_ALU_SHL_MH;
3144                 if (dst[0] == 'h' && src[0] == 'h')
3145                         instr->type = INSTR_ALU_SHL_HH;
3146
3147                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3148                 instr->alu.dst.n_bits = fdst->n_bits;
3149                 instr->alu.dst.offset = fdst->offset / 8;
3150                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3151                 instr->alu.src.n_bits = fsrc->n_bits;
3152                 instr->alu.src.offset = fsrc->offset / 8;
3153                 return 0;
3154         }
3155
3156         /* SHL_MI, SHL_HI. */
3157         src_val = strtoul(src, &src, 0);
3158         CHECK(!src[0], EINVAL);
3159
3160         instr->type = INSTR_ALU_SHL_MI;
3161         if (dst[0] == 'h')
3162                 instr->type = INSTR_ALU_SHL_HI;
3163
3164         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3165         instr->alu.dst.n_bits = fdst->n_bits;
3166         instr->alu.dst.offset = fdst->offset / 8;
3167         instr->alu.src_val = (uint32_t)src_val;
3168         return 0;
3169 }
3170
3171 static int
3172 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3173                         struct action *action,
3174                         char **tokens,
3175                         int n_tokens,
3176                         struct instruction *instr,
3177                         struct instruction_data *data __rte_unused)
3178 {
3179         char *dst = tokens[1], *src = tokens[2];
3180         struct field *fdst, *fsrc;
3181         uint32_t dst_struct_id, src_struct_id, src_val;
3182
3183         CHECK(n_tokens == 3, EINVAL);
3184
3185         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3186         CHECK(fdst, EINVAL);
3187
3188         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3189         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3190         if (fsrc) {
3191                 instr->type = INSTR_ALU_SHR;
3192                 if (dst[0] == 'h' && src[0] == 'm')
3193                         instr->type = INSTR_ALU_SHR_HM;
3194                 if (dst[0] == 'm' && src[0] == 'h')
3195                         instr->type = INSTR_ALU_SHR_MH;
3196                 if (dst[0] == 'h' && src[0] == 'h')
3197                         instr->type = INSTR_ALU_SHR_HH;
3198
3199                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3200                 instr->alu.dst.n_bits = fdst->n_bits;
3201                 instr->alu.dst.offset = fdst->offset / 8;
3202                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3203                 instr->alu.src.n_bits = fsrc->n_bits;
3204                 instr->alu.src.offset = fsrc->offset / 8;
3205                 return 0;
3206         }
3207
3208         /* SHR_MI, SHR_HI. */
3209         src_val = strtoul(src, &src, 0);
3210         CHECK(!src[0], EINVAL);
3211
3212         instr->type = INSTR_ALU_SHR_MI;
3213         if (dst[0] == 'h')
3214                 instr->type = INSTR_ALU_SHR_HI;
3215
3216         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3217         instr->alu.dst.n_bits = fdst->n_bits;
3218         instr->alu.dst.offset = fdst->offset / 8;
3219         instr->alu.src_val = (uint32_t)src_val;
3220         return 0;
3221 }
3222
3223 static int
3224 instr_alu_and_translate(struct rte_swx_pipeline *p,
3225                         struct action *action,
3226                         char **tokens,
3227                         int n_tokens,
3228                         struct instruction *instr,
3229                         struct instruction_data *data __rte_unused)
3230 {
3231         char *dst = tokens[1], *src = tokens[2];
3232         struct field *fdst, *fsrc;
3233         uint32_t dst_struct_id, src_struct_id, src_val;
3234
3235         CHECK(n_tokens == 3, EINVAL);
3236
3237         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3238         CHECK(fdst, EINVAL);
3239
3240         /* AND or AND_S. */
3241         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3242         if (fsrc) {
3243                 instr->type = INSTR_ALU_AND;
3244                 if ((dst[0] == 'h' && src[0] != 'h') ||
3245                     (dst[0] != 'h' && src[0] == 'h'))
3246                         instr->type = INSTR_ALU_AND_S;
3247
3248                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3249                 instr->alu.dst.n_bits = fdst->n_bits;
3250                 instr->alu.dst.offset = fdst->offset / 8;
3251                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3252                 instr->alu.src.n_bits = fsrc->n_bits;
3253                 instr->alu.src.offset = fsrc->offset / 8;
3254                 return 0;
3255         }
3256
3257         /* AND_I. */
3258         src_val = strtoul(src, &src, 0);
3259         CHECK(!src[0], EINVAL);
3260
3261         if (dst[0] == 'h')
3262                 src_val = htonl(src_val);
3263
3264         instr->type = INSTR_ALU_AND_I;
3265         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3266         instr->alu.dst.n_bits = fdst->n_bits;
3267         instr->alu.dst.offset = fdst->offset / 8;
3268         instr->alu.src_val = (uint32_t)src_val;
3269         return 0;
3270 }
3271
3272 static int
3273 instr_alu_or_translate(struct rte_swx_pipeline *p,
3274                        struct action *action,
3275                        char **tokens,
3276                        int n_tokens,
3277                        struct instruction *instr,
3278                        struct instruction_data *data __rte_unused)
3279 {
3280         char *dst = tokens[1], *src = tokens[2];
3281         struct field *fdst, *fsrc;
3282         uint32_t dst_struct_id, src_struct_id, src_val;
3283
3284         CHECK(n_tokens == 3, EINVAL);
3285
3286         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3287         CHECK(fdst, EINVAL);
3288
3289         /* OR or OR_S. */
3290         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3291         if (fsrc) {
3292                 instr->type = INSTR_ALU_OR;
3293                 if ((dst[0] == 'h' && src[0] != 'h') ||
3294                     (dst[0] != 'h' && src[0] == 'h'))
3295                         instr->type = INSTR_ALU_OR_S;
3296
3297                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3298                 instr->alu.dst.n_bits = fdst->n_bits;
3299                 instr->alu.dst.offset = fdst->offset / 8;
3300                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3301                 instr->alu.src.n_bits = fsrc->n_bits;
3302                 instr->alu.src.offset = fsrc->offset / 8;
3303                 return 0;
3304         }
3305
3306         /* OR_I. */
3307         src_val = strtoul(src, &src, 0);
3308         CHECK(!src[0], EINVAL);
3309
3310         if (dst[0] == 'h')
3311                 src_val = htonl(src_val);
3312
3313         instr->type = INSTR_ALU_OR_I;
3314         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3315         instr->alu.dst.n_bits = fdst->n_bits;
3316         instr->alu.dst.offset = fdst->offset / 8;
3317         instr->alu.src_val = (uint32_t)src_val;
3318         return 0;
3319 }
3320
3321 static int
3322 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3323                         struct action *action,
3324                         char **tokens,
3325                         int n_tokens,
3326                         struct instruction *instr,
3327                         struct instruction_data *data __rte_unused)
3328 {
3329         char *dst = tokens[1], *src = tokens[2];
3330         struct field *fdst, *fsrc;
3331         uint32_t dst_struct_id, src_struct_id, src_val;
3332
3333         CHECK(n_tokens == 3, EINVAL);
3334
3335         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3336         CHECK(fdst, EINVAL);
3337
3338         /* XOR or XOR_S. */
3339         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3340         if (fsrc) {
3341                 instr->type = INSTR_ALU_XOR;
3342                 if ((dst[0] == 'h' && src[0] != 'h') ||
3343                     (dst[0] != 'h' && src[0] == 'h'))
3344                         instr->type = INSTR_ALU_XOR_S;
3345
3346                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3347                 instr->alu.dst.n_bits = fdst->n_bits;
3348                 instr->alu.dst.offset = fdst->offset / 8;
3349                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3350                 instr->alu.src.n_bits = fsrc->n_bits;
3351                 instr->alu.src.offset = fsrc->offset / 8;
3352                 return 0;
3353         }
3354
3355         /* XOR_I. */
3356         src_val = strtoul(src, &src, 0);
3357         CHECK(!src[0], EINVAL);
3358
3359         if (dst[0] == 'h')
3360                 src_val = htonl(src_val);
3361
3362         instr->type = INSTR_ALU_XOR_I;
3363         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3364         instr->alu.dst.n_bits = fdst->n_bits;
3365         instr->alu.dst.offset = fdst->offset / 8;
3366         instr->alu.src_val = (uint32_t)src_val;
3367         return 0;
3368 }
3369
3370 static inline void
3371 instr_alu_add_exec(struct rte_swx_pipeline *p)
3372 {
3373         struct thread *t = &p->threads[p->thread_id];
3374         struct instruction *ip = t->ip;
3375
3376         TRACE("[Thread %2u] add\n", p->thread_id);
3377
3378         /* Structs. */
3379         ALU(t, ip, +);
3380
3381         /* Thread. */
3382         thread_ip_inc(p);
3383 }
3384
3385 static inline void
3386 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3387 {
3388         struct thread *t = &p->threads[p->thread_id];
3389         struct instruction *ip = t->ip;
3390
3391         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3392
3393         /* Structs. */
3394         ALU_MH(t, ip, +);
3395
3396         /* Thread. */
3397         thread_ip_inc(p);
3398 }
3399
3400 static inline void
3401 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3402 {
3403         struct thread *t = &p->threads[p->thread_id];
3404         struct instruction *ip = t->ip;
3405
3406         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3407
3408         /* Structs. */
3409         ALU_HM(t, ip, +);
3410
3411         /* Thread. */
3412         thread_ip_inc(p);
3413 }
3414
3415 static inline void
3416 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3417 {
3418         struct thread *t = &p->threads[p->thread_id];
3419         struct instruction *ip = t->ip;
3420
3421         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3422
3423         /* Structs. */
3424         ALU_HH(t, ip, +);
3425
3426         /* Thread. */
3427         thread_ip_inc(p);
3428 }
3429
3430 static inline void
3431 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3432 {
3433         struct thread *t = &p->threads[p->thread_id];
3434         struct instruction *ip = t->ip;
3435
3436         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3437
3438         /* Structs. */
3439         ALU_MI(t, ip, +);
3440
3441         /* Thread. */
3442         thread_ip_inc(p);
3443 }
3444
3445 static inline void
3446 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3447 {
3448         struct thread *t = &p->threads[p->thread_id];
3449         struct instruction *ip = t->ip;
3450
3451         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3452
3453         /* Structs. */
3454         ALU_HI(t, ip, +);
3455
3456         /* Thread. */
3457         thread_ip_inc(p);
3458 }
3459
3460 static inline void
3461 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3462 {
3463         struct thread *t = &p->threads[p->thread_id];
3464         struct instruction *ip = t->ip;
3465
3466         TRACE("[Thread %2u] sub\n", p->thread_id);
3467
3468         /* Structs. */
3469         ALU(t, ip, -);
3470
3471         /* Thread. */
3472         thread_ip_inc(p);
3473 }
3474
3475 static inline void
3476 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3477 {
3478         struct thread *t = &p->threads[p->thread_id];
3479         struct instruction *ip = t->ip;
3480
3481         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3482
3483         /* Structs. */
3484         ALU_MH(t, ip, -);
3485
3486         /* Thread. */
3487         thread_ip_inc(p);
3488 }
3489
3490 static inline void
3491 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3492 {
3493         struct thread *t = &p->threads[p->thread_id];
3494         struct instruction *ip = t->ip;
3495
3496         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3497
3498         /* Structs. */
3499         ALU_HM(t, ip, -);
3500
3501         /* Thread. */
3502         thread_ip_inc(p);
3503 }
3504
3505 static inline void
3506 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3507 {
3508         struct thread *t = &p->threads[p->thread_id];
3509         struct instruction *ip = t->ip;
3510
3511         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3512
3513         /* Structs. */
3514         ALU_HH(t, ip, -);
3515
3516         /* Thread. */
3517         thread_ip_inc(p);
3518 }
3519
3520 static inline void
3521 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3522 {
3523         struct thread *t = &p->threads[p->thread_id];
3524         struct instruction *ip = t->ip;
3525
3526         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3527
3528         /* Structs. */
3529         ALU_MI(t, ip, -);
3530
3531         /* Thread. */
3532         thread_ip_inc(p);
3533 }
3534
3535 static inline void
3536 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3537 {
3538         struct thread *t = &p->threads[p->thread_id];
3539         struct instruction *ip = t->ip;
3540
3541         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3542
3543         /* Structs. */
3544         ALU_HI(t, ip, -);
3545
3546         /* Thread. */
3547         thread_ip_inc(p);
3548 }
3549
3550 static inline void
3551 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3552 {
3553         struct thread *t = &p->threads[p->thread_id];
3554         struct instruction *ip = t->ip;
3555
3556         TRACE("[Thread %2u] shl\n", p->thread_id);
3557
3558         /* Structs. */
3559         ALU(t, ip, <<);
3560
3561         /* Thread. */
3562         thread_ip_inc(p);
3563 }
3564
3565 static inline void
3566 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3567 {
3568         struct thread *t = &p->threads[p->thread_id];
3569         struct instruction *ip = t->ip;
3570
3571         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
3572
3573         /* Structs. */
3574         ALU_MH(t, ip, <<);
3575
3576         /* Thread. */
3577         thread_ip_inc(p);
3578 }
3579
3580 static inline void
3581 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3582 {
3583         struct thread *t = &p->threads[p->thread_id];
3584         struct instruction *ip = t->ip;
3585
3586         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
3587
3588         /* Structs. */
3589         ALU_HM(t, ip, <<);
3590
3591         /* Thread. */
3592         thread_ip_inc(p);
3593 }
3594
3595 static inline void
3596 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3597 {
3598         struct thread *t = &p->threads[p->thread_id];
3599         struct instruction *ip = t->ip;
3600
3601         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
3602
3603         /* Structs. */
3604         ALU_HH(t, ip, <<);
3605
3606         /* Thread. */
3607         thread_ip_inc(p);
3608 }
3609
3610 static inline void
3611 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3612 {
3613         struct thread *t = &p->threads[p->thread_id];
3614         struct instruction *ip = t->ip;
3615
3616         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
3617
3618         /* Structs. */
3619         ALU_MI(t, ip, <<);
3620
3621         /* Thread. */
3622         thread_ip_inc(p);
3623 }
3624
3625 static inline void
3626 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3627 {
3628         struct thread *t = &p->threads[p->thread_id];
3629         struct instruction *ip = t->ip;
3630
3631         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
3632
3633         /* Structs. */
3634         ALU_HI(t, ip, <<);
3635
3636         /* Thread. */
3637         thread_ip_inc(p);
3638 }
3639
3640 static inline void
3641 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3642 {
3643         struct thread *t = &p->threads[p->thread_id];
3644         struct instruction *ip = t->ip;
3645
3646         TRACE("[Thread %2u] shr\n", p->thread_id);
3647
3648         /* Structs. */
3649         ALU(t, ip, >>);
3650
3651         /* Thread. */
3652         thread_ip_inc(p);
3653 }
3654
3655 static inline void
3656 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3657 {
3658         struct thread *t = &p->threads[p->thread_id];
3659         struct instruction *ip = t->ip;
3660
3661         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
3662
3663         /* Structs. */
3664         ALU_MH(t, ip, >>);
3665
3666         /* Thread. */
3667         thread_ip_inc(p);
3668 }
3669
3670 static inline void
3671 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3672 {
3673         struct thread *t = &p->threads[p->thread_id];
3674         struct instruction *ip = t->ip;
3675
3676         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
3677
3678         /* Structs. */
3679         ALU_HM(t, ip, >>);
3680
3681         /* Thread. */
3682         thread_ip_inc(p);
3683 }
3684
3685 static inline void
3686 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3687 {
3688         struct thread *t = &p->threads[p->thread_id];
3689         struct instruction *ip = t->ip;
3690
3691         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
3692
3693         /* Structs. */
3694         ALU_HH(t, ip, >>);
3695
3696         /* Thread. */
3697         thread_ip_inc(p);
3698 }
3699
3700 static inline void
3701 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3702 {
3703         struct thread *t = &p->threads[p->thread_id];
3704         struct instruction *ip = t->ip;
3705
3706         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
3707
3708         /* Structs. */
3709         ALU_MI(t, ip, >>);
3710
3711         /* Thread. */
3712         thread_ip_inc(p);
3713 }
3714
3715 static inline void
3716 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3717 {
3718         struct thread *t = &p->threads[p->thread_id];
3719         struct instruction *ip = t->ip;
3720
3721         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
3722
3723         /* Structs. */
3724         ALU_HI(t, ip, >>);
3725
3726         /* Thread. */
3727         thread_ip_inc(p);
3728 }
3729
3730 static inline void
3731 instr_alu_and_exec(struct rte_swx_pipeline *p)
3732 {
3733         struct thread *t = &p->threads[p->thread_id];
3734         struct instruction *ip = t->ip;
3735
3736         TRACE("[Thread %2u] and\n", p->thread_id);
3737
3738         /* Structs. */
3739         ALU(t, ip, &);
3740
3741         /* Thread. */
3742         thread_ip_inc(p);
3743 }
3744
3745 static inline void
3746 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
3747 {
3748         struct thread *t = &p->threads[p->thread_id];
3749         struct instruction *ip = t->ip;
3750
3751         TRACE("[Thread %2u] and (s)\n", p->thread_id);
3752
3753         /* Structs. */
3754         ALU_S(t, ip, &);
3755
3756         /* Thread. */
3757         thread_ip_inc(p);
3758 }
3759
3760 static inline void
3761 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3762 {
3763         struct thread *t = &p->threads[p->thread_id];
3764         struct instruction *ip = t->ip;
3765
3766         TRACE("[Thread %2u] and (i)\n", p->thread_id);
3767
3768         /* Structs. */
3769         ALU_I(t, ip, &);
3770
3771         /* Thread. */
3772         thread_ip_inc(p);
3773 }
3774
3775 static inline void
3776 instr_alu_or_exec(struct rte_swx_pipeline *p)
3777 {
3778         struct thread *t = &p->threads[p->thread_id];
3779         struct instruction *ip = t->ip;
3780
3781         TRACE("[Thread %2u] or\n", p->thread_id);
3782
3783         /* Structs. */
3784         ALU(t, ip, |);
3785
3786         /* Thread. */
3787         thread_ip_inc(p);
3788 }
3789
3790 static inline void
3791 instr_alu_or_s_exec(struct rte_swx_pipeline *p)
3792 {
3793         struct thread *t = &p->threads[p->thread_id];
3794         struct instruction *ip = t->ip;
3795
3796         TRACE("[Thread %2u] or (s)\n", p->thread_id);
3797
3798         /* Structs. */
3799         ALU_S(t, ip, |);
3800
3801         /* Thread. */
3802         thread_ip_inc(p);
3803 }
3804
3805 static inline void
3806 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3807 {
3808         struct thread *t = &p->threads[p->thread_id];
3809         struct instruction *ip = t->ip;
3810
3811         TRACE("[Thread %2u] or (i)\n", p->thread_id);
3812
3813         /* Structs. */
3814         ALU_I(t, ip, |);
3815
3816         /* Thread. */
3817         thread_ip_inc(p);
3818 }
3819
3820 static inline void
3821 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3822 {
3823         struct thread *t = &p->threads[p->thread_id];
3824         struct instruction *ip = t->ip;
3825
3826         TRACE("[Thread %2u] xor\n", p->thread_id);
3827
3828         /* Structs. */
3829         ALU(t, ip, ^);
3830
3831         /* Thread. */
3832         thread_ip_inc(p);
3833 }
3834
3835 static inline void
3836 instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
3837 {
3838         struct thread *t = &p->threads[p->thread_id];
3839         struct instruction *ip = t->ip;
3840
3841         TRACE("[Thread %2u] xor (s)\n", p->thread_id);
3842
3843         /* Structs. */
3844         ALU_S(t, ip, ^);
3845
3846         /* Thread. */
3847         thread_ip_inc(p);
3848 }
3849
3850 static inline void
3851 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3852 {
3853         struct thread *t = &p->threads[p->thread_id];
3854         struct instruction *ip = t->ip;
3855
3856         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3857
3858         /* Structs. */
3859         ALU_I(t, ip, ^);
3860
3861         /* Thread. */
3862         thread_ip_inc(p);
3863 }
3864
3865 static inline void
3866 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3867 {
3868         struct thread *t = &p->threads[p->thread_id];
3869         struct instruction *ip = t->ip;
3870         uint8_t *dst_struct, *src_struct;
3871         uint16_t *dst16_ptr, dst;
3872         uint64_t *src64_ptr, src64, src64_mask, src;
3873         uint64_t r;
3874
3875         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3876
3877         /* Structs. */
3878         dst_struct = t->structs[ip->alu.dst.struct_id];
3879         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3880         dst = *dst16_ptr;
3881
3882         src_struct = t->structs[ip->alu.src.struct_id];
3883         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3884         src64 = *src64_ptr;
3885         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3886         src = src64 & src64_mask;
3887
3888         r = dst;
3889         r = ~r & 0xFFFF;
3890
3891         /* The first input (r) is a 16-bit number. The second and the third
3892          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3893          * three numbers (output r) is a 34-bit number.
3894          */
3895         r += (src >> 32) + (src & 0xFFFFFFFF);
3896
3897         /* The first input is a 16-bit number. The second input is an 18-bit
3898          * number. In the worst case scenario, the sum of the two numbers is a
3899          * 19-bit number.
3900          */
3901         r = (r & 0xFFFF) + (r >> 16);
3902
3903         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3904          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3905          */
3906         r = (r & 0xFFFF) + (r >> 16);
3907
3908         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3909          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3910          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3911          * therefore the output r is always a 16-bit number.
3912          */
3913         r = (r & 0xFFFF) + (r >> 16);
3914
3915         r = ~r & 0xFFFF;
3916         r = r ? r : 0xFFFF;
3917
3918         *dst16_ptr = (uint16_t)r;
3919
3920         /* Thread. */
3921         thread_ip_inc(p);
3922 }
3923
3924 static inline void
3925 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3926 {
3927         struct thread *t = &p->threads[p->thread_id];
3928         struct instruction *ip = t->ip;
3929         uint8_t *dst_struct, *src_struct;
3930         uint16_t *dst16_ptr, dst;
3931         uint64_t *src64_ptr, src64, src64_mask, src;
3932         uint64_t r;
3933
3934         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3935
3936         /* Structs. */
3937         dst_struct = t->structs[ip->alu.dst.struct_id];
3938         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3939         dst = *dst16_ptr;
3940
3941         src_struct = t->structs[ip->alu.src.struct_id];
3942         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3943         src64 = *src64_ptr;
3944         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3945         src = src64 & src64_mask;
3946
3947         r = dst;
3948         r = ~r & 0xFFFF;
3949
3950         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3951          * the following sequence of operations in 2's complement arithmetic:
3952          *    a '- b = (a - b) % 0xFFFF.
3953          *
3954          * In order to prevent an underflow for the below subtraction, in which
3955          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3956          * minuend), we first add a multiple of the 0xFFFF modulus to the
3957          * minuend. The number we add to the minuend needs to be a 34-bit number
3958          * or higher, so for readability reasons we picked the 36-bit multiple.
3959          * We are effectively turning the 16-bit minuend into a 36-bit number:
3960          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3961          */
3962         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3963
3964         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3965          * result (the output r) is a 36-bit number.
3966          */
3967         r -= (src >> 32) + (src & 0xFFFFFFFF);
3968
3969         /* The first input is a 16-bit number. The second input is a 20-bit
3970          * number. Their sum is a 21-bit number.
3971          */
3972         r = (r & 0xFFFF) + (r >> 16);
3973
3974         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3975          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3976          */
3977         r = (r & 0xFFFF) + (r >> 16);
3978
3979         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3980          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3981          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3982          * generated, therefore the output r is always a 16-bit number.
3983          */
3984         r = (r & 0xFFFF) + (r >> 16);
3985
3986         r = ~r & 0xFFFF;
3987         r = r ? r : 0xFFFF;
3988
3989         *dst16_ptr = (uint16_t)r;
3990
3991         /* Thread. */
3992         thread_ip_inc(p);
3993 }
3994
3995 static inline void
3996 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3997 {
3998         struct thread *t = &p->threads[p->thread_id];
3999         struct instruction *ip = t->ip;
4000         uint8_t *dst_struct, *src_struct;
4001         uint16_t *dst16_ptr;
4002         uint32_t *src32_ptr;
4003         uint64_t r0, r1;
4004
4005         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
4006
4007         /* Structs. */
4008         dst_struct = t->structs[ip->alu.dst.struct_id];
4009         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4010
4011         src_struct = t->structs[ip->alu.src.struct_id];
4012         src32_ptr = (uint32_t *)&src_struct[0];
4013
4014         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
4015         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
4016         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
4017         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
4018         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
4019
4020         /* The first input is a 16-bit number. The second input is a 19-bit
4021          * number. Their sum is a 20-bit number.
4022          */
4023         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4024
4025         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4026          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
4027          */
4028         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4029
4030         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4031          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4032          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
4033          * generated, therefore the output r is always a 16-bit number.
4034          */
4035         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4036
4037         r0 = ~r0 & 0xFFFF;
4038         r0 = r0 ? r0 : 0xFFFF;
4039
4040         *dst16_ptr = (uint16_t)r0;
4041
4042         /* Thread. */
4043         thread_ip_inc(p);
4044 }
4045
4046 static inline void
4047 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4048 {
4049         struct thread *t = &p->threads[p->thread_id];
4050         struct instruction *ip = t->ip;
4051         uint8_t *dst_struct, *src_struct;
4052         uint16_t *dst16_ptr;
4053         uint32_t *src32_ptr;
4054         uint64_t r = 0;
4055         uint32_t i;
4056
4057         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
4058
4059         /* Structs. */
4060         dst_struct = t->structs[ip->alu.dst.struct_id];
4061         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4062
4063         src_struct = t->structs[ip->alu.src.struct_id];
4064         src32_ptr = (uint32_t *)&src_struct[0];
4065
4066         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
4067          * Therefore, in the worst case scenario, a 35-bit number is added to a
4068          * 16-bit number (the input r), so the output r is 36-bit number.
4069          */
4070         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
4071                 r += *src32_ptr;
4072
4073         /* The first input is a 16-bit number. The second input is a 20-bit
4074          * number. Their sum is a 21-bit number.
4075          */
4076         r = (r & 0xFFFF) + (r >> 16);
4077
4078         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4079          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
4080          */
4081         r = (r & 0xFFFF) + (r >> 16);
4082
4083         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4084          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4085          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4086          * generated, therefore the output r is always a 16-bit number.
4087          */
4088         r = (r & 0xFFFF) + (r >> 16);
4089
4090         r = ~r & 0xFFFF;
4091         r = r ? r : 0xFFFF;
4092
4093         *dst16_ptr = (uint16_t)r;
4094
4095         /* Thread. */
4096         thread_ip_inc(p);
4097 }
4098
4099 #define RTE_SWX_INSTRUCTION_TOKENS_MAX 16
4100
4101 static int
4102 instr_translate(struct rte_swx_pipeline *p,
4103                 struct action *action,
4104                 char *string,
4105                 struct instruction *instr,
4106                 struct instruction_data *data)
4107 {
4108         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
4109         int n_tokens = 0, tpos = 0;
4110
4111         /* Parse the instruction string into tokens. */
4112         for ( ; ; ) {
4113                 char *token;
4114
4115                 token = strtok_r(string, " \t\v", &string);
4116                 if (!token)
4117                         break;
4118
4119                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
4120
4121                 tokens[n_tokens] = token;
4122                 n_tokens++;
4123         }
4124
4125         CHECK(n_tokens, EINVAL);
4126
4127         /* Handle the optional instruction label. */
4128         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
4129                 strcpy(data->label, tokens[0]);
4130
4131                 tpos += 2;
4132                 CHECK(n_tokens - tpos, EINVAL);
4133         }
4134
4135         /* Identify the instruction type. */
4136         if (!strcmp(tokens[tpos], "rx"))
4137                 return instr_rx_translate(p,
4138                                           action,
4139                                           &tokens[tpos],
4140                                           n_tokens - tpos,
4141                                           instr,
4142                                           data);
4143
4144         if (!strcmp(tokens[tpos], "tx"))
4145                 return instr_tx_translate(p,
4146                                           action,
4147                                           &tokens[tpos],
4148                                           n_tokens - tpos,
4149                                           instr,
4150                                           data);
4151
4152         if (!strcmp(tokens[tpos], "extract"))
4153                 return instr_hdr_extract_translate(p,
4154                                                    action,
4155                                                    &tokens[tpos],
4156                                                    n_tokens - tpos,
4157                                                    instr,
4158                                                    data);
4159
4160         if (!strcmp(tokens[tpos], "emit"))
4161                 return instr_hdr_emit_translate(p,
4162                                                 action,
4163                                                 &tokens[tpos],
4164                                                 n_tokens - tpos,
4165                                                 instr,
4166                                                 data);
4167
4168         if (!strcmp(tokens[tpos], "validate"))
4169                 return instr_hdr_validate_translate(p,
4170                                                     action,
4171                                                     &tokens[tpos],
4172                                                     n_tokens - tpos,
4173                                                     instr,
4174                                                     data);
4175
4176         if (!strcmp(tokens[tpos], "invalidate"))
4177                 return instr_hdr_invalidate_translate(p,
4178                                                       action,
4179                                                       &tokens[tpos],
4180                                                       n_tokens - tpos,
4181                                                       instr,
4182                                                       data);
4183
4184         if (!strcmp(tokens[tpos], "mov"))
4185                 return instr_mov_translate(p,
4186                                            action,
4187                                            &tokens[tpos],
4188                                            n_tokens - tpos,
4189                                            instr,
4190                                            data);
4191
4192         if (!strcmp(tokens[tpos], "dma"))
4193                 return instr_dma_translate(p,
4194                                            action,
4195                                            &tokens[tpos],
4196                                            n_tokens - tpos,
4197                                            instr,
4198                                            data);
4199
4200         if (!strcmp(tokens[tpos], "add"))
4201                 return instr_alu_add_translate(p,
4202                                                action,
4203                                                &tokens[tpos],
4204                                                n_tokens - tpos,
4205                                                instr,
4206                                                data);
4207
4208         if (!strcmp(tokens[tpos], "sub"))
4209                 return instr_alu_sub_translate(p,
4210                                                action,
4211                                                &tokens[tpos],
4212                                                n_tokens - tpos,
4213                                                instr,
4214                                                data);
4215
4216         if (!strcmp(tokens[tpos], "ckadd"))
4217                 return instr_alu_ckadd_translate(p,
4218                                                  action,
4219                                                  &tokens[tpos],
4220                                                  n_tokens - tpos,
4221                                                  instr,
4222                                                  data);
4223
4224         if (!strcmp(tokens[tpos], "cksub"))
4225                 return instr_alu_cksub_translate(p,
4226                                                  action,
4227                                                  &tokens[tpos],
4228                                                  n_tokens - tpos,
4229                                                  instr,
4230                                                  data);
4231
4232         if (!strcmp(tokens[tpos], "and"))
4233                 return instr_alu_and_translate(p,
4234                                                action,
4235                                                &tokens[tpos],
4236                                                n_tokens - tpos,
4237                                                instr,
4238                                                data);
4239
4240         if (!strcmp(tokens[tpos], "or"))
4241                 return instr_alu_or_translate(p,
4242                                               action,
4243                                               &tokens[tpos],
4244                                               n_tokens - tpos,
4245                                               instr,
4246                                               data);
4247
4248         if (!strcmp(tokens[tpos], "xor"))
4249                 return instr_alu_xor_translate(p,
4250                                                action,
4251                                                &tokens[tpos],
4252                                                n_tokens - tpos,
4253                                                instr,
4254                                                data);
4255
4256         if (!strcmp(tokens[tpos], "shl"))
4257                 return instr_alu_shl_translate(p,
4258                                                action,
4259                                                &tokens[tpos],
4260                                                n_tokens - tpos,
4261                                                instr,
4262                                                data);
4263
4264         if (!strcmp(tokens[tpos], "shr"))
4265                 return instr_alu_shr_translate(p,
4266                                                action,
4267                                                &tokens[tpos],
4268                                                n_tokens - tpos,
4269                                                instr,
4270                                                data);
4271
4272         CHECK(0, EINVAL);
4273 }
4274
4275 static uint32_t
4276 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
4277 {
4278         uint32_t count = 0, i;
4279
4280         if (!label[0])
4281                 return 0;
4282
4283         for (i = 0; i < n; i++)
4284                 if (!strcmp(label, data[i].jmp_label))
4285                         count++;
4286
4287         return count;
4288 }
4289
4290 static int
4291 instr_label_check(struct instruction_data *instruction_data,
4292                   uint32_t n_instructions)
4293 {
4294         uint32_t i;
4295
4296         /* Check that all instruction labels are unique. */
4297         for (i = 0; i < n_instructions; i++) {
4298                 struct instruction_data *data = &instruction_data[i];
4299                 char *label = data->label;
4300                 uint32_t j;
4301
4302                 if (!label[0])
4303                         continue;
4304
4305                 for (j = i + 1; j < n_instructions; j++)
4306                         CHECK(strcmp(label, data[j].label), EINVAL);
4307         }
4308
4309         /* Get users for each instruction label. */
4310         for (i = 0; i < n_instructions; i++) {
4311                 struct instruction_data *data = &instruction_data[i];
4312                 char *label = data->label;
4313
4314                 data->n_users = label_is_used(instruction_data,
4315                                               n_instructions,
4316                                               label);
4317         }
4318
4319         return 0;
4320 }
4321
4322 static int
4323 instruction_config(struct rte_swx_pipeline *p,
4324                    struct action *a,
4325                    const char **instructions,
4326                    uint32_t n_instructions)
4327 {
4328         struct instruction *instr = NULL;
4329         struct instruction_data *data = NULL;
4330         char *string = NULL;
4331         int err = 0;
4332         uint32_t i;
4333
4334         CHECK(n_instructions, EINVAL);
4335         CHECK(instructions, EINVAL);
4336         for (i = 0; i < n_instructions; i++)
4337                 CHECK(instructions[i], EINVAL);
4338
4339         /* Memory allocation. */
4340         instr = calloc(n_instructions, sizeof(struct instruction));
4341         if (!instr) {
4342                 err = ENOMEM;
4343                 goto error;
4344         }
4345
4346         data = calloc(n_instructions, sizeof(struct instruction_data));
4347         if (!data) {
4348                 err = ENOMEM;
4349                 goto error;
4350         }
4351
4352         for (i = 0; i < n_instructions; i++) {
4353                 string = strdup(instructions[i]);
4354                 if (!string) {
4355                         err = ENOMEM;
4356                         goto error;
4357                 }
4358
4359                 err = instr_translate(p, a, string, &instr[i], &data[i]);
4360                 if (err)
4361                         goto error;
4362
4363                 free(string);
4364         }
4365
4366         err = instr_label_check(data, n_instructions);
4367         if (err)
4368                 goto error;
4369
4370         free(data);
4371
4372         if (a) {
4373                 a->instructions = instr;
4374                 a->n_instructions = n_instructions;
4375         } else {
4376                 p->instructions = instr;
4377                 p->n_instructions = n_instructions;
4378         }
4379
4380         return 0;
4381
4382 error:
4383         free(string);
4384         free(data);
4385         free(instr);
4386         return err;
4387 }
4388
4389 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
4390
4391 static instr_exec_t instruction_table[] = {
4392         [INSTR_RX] = instr_rx_exec,
4393         [INSTR_TX] = instr_tx_exec,
4394
4395         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
4396         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
4397         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
4398         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
4399         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
4400         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
4401         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
4402         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
4403
4404         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
4405         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
4406         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
4407         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
4408         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
4409         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
4410         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
4411         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
4412         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
4413
4414         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
4415         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
4416
4417         [INSTR_MOV] = instr_mov_exec,
4418         [INSTR_MOV_S] = instr_mov_s_exec,
4419         [INSTR_MOV_I] = instr_mov_i_exec,
4420
4421         [INSTR_DMA_HT] = instr_dma_ht_exec,
4422         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
4423         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
4424         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
4425         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
4426         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
4427         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
4428         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
4429
4430         [INSTR_ALU_ADD] = instr_alu_add_exec,
4431         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
4432         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
4433         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
4434         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
4435         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
4436
4437         [INSTR_ALU_SUB] = instr_alu_sub_exec,
4438         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
4439         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
4440         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
4441         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
4442         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
4443
4444         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
4445         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
4446         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
4447         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
4448
4449         [INSTR_ALU_AND] = instr_alu_and_exec,
4450         [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
4451         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
4452
4453         [INSTR_ALU_OR] = instr_alu_or_exec,
4454         [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
4455         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
4456
4457         [INSTR_ALU_XOR] = instr_alu_xor_exec,
4458         [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec,
4459         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
4460
4461         [INSTR_ALU_SHL] = instr_alu_shl_exec,
4462         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
4463         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
4464         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
4465         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
4466         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
4467
4468         [INSTR_ALU_SHR] = instr_alu_shr_exec,
4469         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
4470         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
4471         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
4472         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
4473         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
4474 };
4475
4476 static inline void
4477 instr_exec(struct rte_swx_pipeline *p)
4478 {
4479         struct thread *t = &p->threads[p->thread_id];
4480         struct instruction *ip = t->ip;
4481         instr_exec_t instr = instruction_table[ip->type];
4482
4483         instr(p);
4484 }
4485
4486 /*
4487  * Action.
4488  */
4489 static struct action *
4490 action_find(struct rte_swx_pipeline *p, const char *name)
4491 {
4492         struct action *elem;
4493
4494         if (!name)
4495                 return NULL;
4496
4497         TAILQ_FOREACH(elem, &p->actions, node)
4498                 if (strcmp(elem->name, name) == 0)
4499                         return elem;
4500
4501         return NULL;
4502 }
4503
4504 static struct field *
4505 action_field_find(struct action *a, const char *name)
4506 {
4507         return a->st ? struct_type_field_find(a->st, name) : NULL;
4508 }
4509
4510 static struct field *
4511 action_field_parse(struct action *action, const char *name)
4512 {
4513         if (name[0] != 't' || name[1] != '.')
4514                 return NULL;
4515
4516         return action_field_find(action, &name[2]);
4517 }
4518
4519 int
4520 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
4521                                const char *name,
4522                                const char *args_struct_type_name,
4523                                const char **instructions,
4524                                uint32_t n_instructions)
4525 {
4526         struct struct_type *args_struct_type;
4527         struct action *a;
4528         int err;
4529
4530         CHECK(p, EINVAL);
4531
4532         CHECK_NAME(name, EINVAL);
4533         CHECK(!action_find(p, name), EEXIST);
4534
4535         if (args_struct_type_name) {
4536                 CHECK_NAME(args_struct_type_name, EINVAL);
4537                 args_struct_type = struct_type_find(p, args_struct_type_name);
4538                 CHECK(args_struct_type, EINVAL);
4539         } else {
4540                 args_struct_type = NULL;
4541         }
4542
4543         /* Node allocation. */
4544         a = calloc(1, sizeof(struct action));
4545         CHECK(a, ENOMEM);
4546
4547         /* Node initialization. */
4548         strcpy(a->name, name);
4549         a->st = args_struct_type;
4550         a->id = p->n_actions;
4551
4552         /* Instruction translation. */
4553         err = instruction_config(p, a, instructions, n_instructions);
4554         if (err) {
4555                 free(a);
4556                 return err;
4557         }
4558
4559         /* Node add to tailq. */
4560         TAILQ_INSERT_TAIL(&p->actions, a, node);
4561         p->n_actions++;
4562
4563         return 0;
4564 }
4565
4566 static int
4567 action_build(struct rte_swx_pipeline *p)
4568 {
4569         struct action *action;
4570
4571         p->action_instructions = calloc(p->n_actions,
4572                                         sizeof(struct instruction *));
4573         CHECK(p->action_instructions, ENOMEM);
4574
4575         TAILQ_FOREACH(action, &p->actions, node)
4576                 p->action_instructions[action->id] = action->instructions;
4577
4578         return 0;
4579 }
4580
4581 static void
4582 action_build_free(struct rte_swx_pipeline *p)
4583 {
4584         free(p->action_instructions);
4585         p->action_instructions = NULL;
4586 }
4587
4588 static void
4589 action_free(struct rte_swx_pipeline *p)
4590 {
4591         action_build_free(p);
4592
4593         for ( ; ; ) {
4594                 struct action *action;
4595
4596                 action = TAILQ_FIRST(&p->actions);
4597                 if (!action)
4598                         break;
4599
4600                 TAILQ_REMOVE(&p->actions, action, node);
4601                 free(action->instructions);
4602                 free(action);
4603         }
4604 }
4605
4606 /*
4607  * Table.
4608  */
4609 static struct table_type *
4610 table_type_find(struct rte_swx_pipeline *p, const char *name)
4611 {
4612         struct table_type *elem;
4613
4614         TAILQ_FOREACH(elem, &p->table_types, node)
4615                 if (strcmp(elem->name, name) == 0)
4616                         return elem;
4617
4618         return NULL;
4619 }
4620
4621 static struct table_type *
4622 table_type_resolve(struct rte_swx_pipeline *p,
4623                    const char *recommended_type_name,
4624                    enum rte_swx_table_match_type match_type)
4625 {
4626         struct table_type *elem;
4627
4628         /* Only consider the recommended type if the match type is correct. */
4629         if (recommended_type_name)
4630                 TAILQ_FOREACH(elem, &p->table_types, node)
4631                         if (!strcmp(elem->name, recommended_type_name) &&
4632                             (elem->match_type == match_type))
4633                                 return elem;
4634
4635         /* Ignore the recommended type and get the first element with this match
4636          * type.
4637          */
4638         TAILQ_FOREACH(elem, &p->table_types, node)
4639                 if (elem->match_type == match_type)
4640                         return elem;
4641
4642         return NULL;
4643 }
4644
4645 static struct table *
4646 table_find(struct rte_swx_pipeline *p, const char *name)
4647 {
4648         struct table *elem;
4649
4650         TAILQ_FOREACH(elem, &p->tables, node)
4651                 if (strcmp(elem->name, name) == 0)
4652                         return elem;
4653
4654         return NULL;
4655 }
4656
4657 static struct table *
4658 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
4659 {
4660         struct table *table = NULL;
4661
4662         TAILQ_FOREACH(table, &p->tables, node)
4663                 if (table->id == id)
4664                         return table;
4665
4666         return NULL;
4667 }
4668
4669 int
4670 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
4671                                      const char *name,
4672                                      enum rte_swx_table_match_type match_type,
4673                                      struct rte_swx_table_ops *ops)
4674 {
4675         struct table_type *elem;
4676
4677         CHECK(p, EINVAL);
4678
4679         CHECK_NAME(name, EINVAL);
4680         CHECK(!table_type_find(p, name), EEXIST);
4681
4682         CHECK(ops, EINVAL);
4683         CHECK(ops->create, EINVAL);
4684         CHECK(ops->lkp, EINVAL);
4685         CHECK(ops->free, EINVAL);
4686
4687         /* Node allocation. */
4688         elem = calloc(1, sizeof(struct table_type));
4689         CHECK(elem, ENOMEM);
4690
4691         /* Node initialization. */
4692         strcpy(elem->name, name);
4693         elem->match_type = match_type;
4694         memcpy(&elem->ops, ops, sizeof(*ops));
4695
4696         /* Node add to tailq. */
4697         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
4698
4699         return 0;
4700 }
4701
4702 static enum rte_swx_table_match_type
4703 table_match_type_resolve(struct rte_swx_match_field_params *fields,
4704                          uint32_t n_fields)
4705 {
4706         uint32_t i;
4707
4708         for (i = 0; i < n_fields; i++)
4709                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
4710                         break;
4711
4712         if (i == n_fields)
4713                 return RTE_SWX_TABLE_MATCH_EXACT;
4714
4715         if ((i == n_fields - 1) &&
4716             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
4717                 return RTE_SWX_TABLE_MATCH_LPM;
4718
4719         return RTE_SWX_TABLE_MATCH_WILDCARD;
4720 }
4721
4722 int
4723 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
4724                               const char *name,
4725                               struct rte_swx_pipeline_table_params *params,
4726                               const char *recommended_table_type_name,
4727                               const char *args,
4728                               uint32_t size)
4729 {
4730         struct table_type *type;
4731         struct table *t;
4732         struct action *default_action;
4733         struct header *header = NULL;
4734         int is_header = 0;
4735         uint32_t offset_prev = 0, action_data_size_max = 0, i;
4736
4737         CHECK(p, EINVAL);
4738
4739         CHECK_NAME(name, EINVAL);
4740         CHECK(!table_find(p, name), EEXIST);
4741
4742         CHECK(params, EINVAL);
4743
4744         /* Match checks. */
4745         CHECK(!params->n_fields || params->fields, EINVAL);
4746         for (i = 0; i < params->n_fields; i++) {
4747                 struct rte_swx_match_field_params *field = &params->fields[i];
4748                 struct header *h;
4749                 struct field *hf, *mf;
4750                 uint32_t offset;
4751
4752                 CHECK_NAME(field->name, EINVAL);
4753
4754                 hf = header_field_parse(p, field->name, &h);
4755                 mf = metadata_field_parse(p, field->name);
4756                 CHECK(hf || mf, EINVAL);
4757
4758                 offset = hf ? hf->offset : mf->offset;
4759
4760                 if (i == 0) {
4761                         is_header = hf ? 1 : 0;
4762                         header = hf ? h : NULL;
4763                         offset_prev = offset;
4764
4765                         continue;
4766                 }
4767
4768                 CHECK((is_header && hf && (h->id == header->id)) ||
4769                       (!is_header && mf), EINVAL);
4770
4771                 CHECK(offset > offset_prev, EINVAL);
4772                 offset_prev = offset;
4773         }
4774
4775         /* Action checks. */
4776         CHECK(params->n_actions, EINVAL);
4777         CHECK(params->action_names, EINVAL);
4778         for (i = 0; i < params->n_actions; i++) {
4779                 const char *action_name = params->action_names[i];
4780                 struct action *a;
4781                 uint32_t action_data_size;
4782
4783                 CHECK(action_name, EINVAL);
4784
4785                 a = action_find(p, action_name);
4786                 CHECK(a, EINVAL);
4787
4788                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
4789                 if (action_data_size > action_data_size_max)
4790                         action_data_size_max = action_data_size;
4791         }
4792
4793         CHECK(params->default_action_name, EINVAL);
4794         for (i = 0; i < p->n_actions; i++)
4795                 if (!strcmp(params->action_names[i],
4796                             params->default_action_name))
4797                         break;
4798         CHECK(i < params->n_actions, EINVAL);
4799         default_action = action_find(p, params->default_action_name);
4800         CHECK((default_action->st && params->default_action_data) ||
4801               !params->default_action_data, EINVAL);
4802
4803         /* Table type checks. */
4804         if (params->n_fields) {
4805                 enum rte_swx_table_match_type match_type;
4806
4807                 match_type = table_match_type_resolve(params->fields,
4808                                                       params->n_fields);
4809                 type = table_type_resolve(p,
4810                                           recommended_table_type_name,
4811                                           match_type);
4812                 CHECK(type, EINVAL);
4813         } else {
4814                 type = NULL;
4815         }
4816
4817         /* Memory allocation. */
4818         t = calloc(1, sizeof(struct table));
4819         CHECK(t, ENOMEM);
4820
4821         t->fields = calloc(params->n_fields, sizeof(struct match_field));
4822         if (!t->fields) {
4823                 free(t);
4824                 CHECK(0, ENOMEM);
4825         }
4826
4827         t->actions = calloc(params->n_actions, sizeof(struct action *));
4828         if (!t->actions) {
4829                 free(t->fields);
4830                 free(t);
4831                 CHECK(0, ENOMEM);
4832         }
4833
4834         if (action_data_size_max) {
4835                 t->default_action_data = calloc(1, action_data_size_max);
4836                 if (!t->default_action_data) {
4837                         free(t->actions);
4838                         free(t->fields);
4839                         free(t);
4840                         CHECK(0, ENOMEM);
4841                 }
4842         }
4843
4844         /* Node initialization. */
4845         strcpy(t->name, name);
4846         if (args && args[0])
4847                 strcpy(t->args, args);
4848         t->type = type;
4849
4850         for (i = 0; i < params->n_fields; i++) {
4851                 struct rte_swx_match_field_params *field = &params->fields[i];
4852                 struct match_field *f = &t->fields[i];
4853
4854                 f->match_type = field->match_type;
4855                 f->field = is_header ?
4856                         header_field_parse(p, field->name, NULL) :
4857                         metadata_field_parse(p, field->name);
4858         }
4859         t->n_fields = params->n_fields;
4860         t->is_header = is_header;
4861         t->header = header;
4862
4863         for (i = 0; i < params->n_actions; i++)
4864                 t->actions[i] = action_find(p, params->action_names[i]);
4865         t->default_action = default_action;
4866         if (default_action->st)
4867                 memcpy(t->default_action_data,
4868                        params->default_action_data,
4869                        default_action->st->n_bits / 8);
4870         t->n_actions = params->n_actions;
4871         t->default_action_is_const = params->default_action_is_const;
4872         t->action_data_size_max = action_data_size_max;
4873
4874         t->size = size;
4875         t->id = p->n_tables;
4876
4877         /* Node add to tailq. */
4878         TAILQ_INSERT_TAIL(&p->tables, t, node);
4879         p->n_tables++;
4880
4881         return 0;
4882 }
4883
4884 static struct rte_swx_table_params *
4885 table_params_get(struct table *table)
4886 {
4887         struct rte_swx_table_params *params;
4888         struct field *first, *last;
4889         uint8_t *key_mask;
4890         uint32_t key_size, key_offset, action_data_size, i;
4891
4892         /* Memory allocation. */
4893         params = calloc(1, sizeof(struct rte_swx_table_params));
4894         if (!params)
4895                 return NULL;
4896
4897         /* Key offset and size. */
4898         first = table->fields[0].field;
4899         last = table->fields[table->n_fields - 1].field;
4900         key_offset = first->offset / 8;
4901         key_size = (last->offset + last->n_bits - first->offset) / 8;
4902
4903         /* Memory allocation. */
4904         key_mask = calloc(1, key_size);
4905         if (!key_mask) {
4906                 free(params);
4907                 return NULL;
4908         }
4909
4910         /* Key mask. */
4911         for (i = 0; i < table->n_fields; i++) {
4912                 struct field *f = table->fields[i].field;
4913                 uint32_t start = (f->offset - first->offset) / 8;
4914                 size_t size = f->n_bits / 8;
4915
4916                 memset(&key_mask[start], 0xFF, size);
4917         }
4918
4919         /* Action data size. */
4920         action_data_size = 0;
4921         for (i = 0; i < table->n_actions; i++) {
4922                 struct action *action = table->actions[i];
4923                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
4924
4925                 if (ads > action_data_size)
4926                         action_data_size = ads;
4927         }
4928
4929         /* Fill in. */
4930         params->match_type = table->type->match_type;
4931         params->key_size = key_size;
4932         params->key_offset = key_offset;
4933         params->key_mask0 = key_mask;
4934         params->action_data_size = action_data_size;
4935         params->n_keys_max = table->size;
4936
4937         return params;
4938 }
4939
4940 static void
4941 table_params_free(struct rte_swx_table_params *params)
4942 {
4943         if (!params)
4944                 return;
4945
4946         free(params->key_mask0);
4947         free(params);
4948 }
4949
4950 static int
4951 table_state_build(struct rte_swx_pipeline *p)
4952 {
4953         struct table *table;
4954
4955         p->table_state = calloc(p->n_tables,
4956                                 sizeof(struct rte_swx_table_state));
4957         CHECK(p->table_state, ENOMEM);
4958
4959         TAILQ_FOREACH(table, &p->tables, node) {
4960                 struct rte_swx_table_state *ts = &p->table_state[table->id];
4961
4962                 if (table->type) {
4963                         struct rte_swx_table_params *params;
4964
4965                         /* ts->obj. */
4966                         params = table_params_get(table);
4967                         CHECK(params, ENOMEM);
4968
4969                         ts->obj = table->type->ops.create(params,
4970                                 NULL,
4971                                 table->args,
4972                                 p->numa_node);
4973
4974                         table_params_free(params);
4975                         CHECK(ts->obj, ENODEV);
4976                 }
4977
4978                 /* ts->default_action_data. */
4979                 if (table->action_data_size_max) {
4980                         ts->default_action_data =
4981                                 malloc(table->action_data_size_max);
4982                         CHECK(ts->default_action_data, ENOMEM);
4983
4984                         memcpy(ts->default_action_data,
4985                                table->default_action_data,
4986                                table->action_data_size_max);
4987                 }
4988
4989                 /* ts->default_action_id. */
4990                 ts->default_action_id = table->default_action->id;
4991         }
4992
4993         return 0;
4994 }
4995
4996 static void
4997 table_state_build_free(struct rte_swx_pipeline *p)
4998 {
4999         uint32_t i;
5000
5001         if (!p->table_state)
5002                 return;
5003
5004         for (i = 0; i < p->n_tables; i++) {
5005                 struct rte_swx_table_state *ts = &p->table_state[i];
5006                 struct table *table = table_find_by_id(p, i);
5007
5008                 /* ts->obj. */
5009                 if (table->type && ts->obj)
5010                         table->type->ops.free(ts->obj);
5011
5012                 /* ts->default_action_data. */
5013                 free(ts->default_action_data);
5014         }
5015
5016         free(p->table_state);
5017         p->table_state = NULL;
5018 }
5019
5020 static void
5021 table_state_free(struct rte_swx_pipeline *p)
5022 {
5023         table_state_build_free(p);
5024 }
5025
5026 static int
5027 table_stub_lkp(void *table __rte_unused,
5028                void *mailbox __rte_unused,
5029                uint8_t **key __rte_unused,
5030                uint64_t *action_id __rte_unused,
5031                uint8_t **action_data __rte_unused,
5032                int *hit)
5033 {
5034         *hit = 0;
5035         return 1; /* DONE. */
5036 }
5037
5038 static int
5039 table_build(struct rte_swx_pipeline *p)
5040 {
5041         uint32_t i;
5042
5043         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
5044                 struct thread *t = &p->threads[i];
5045                 struct table *table;
5046
5047                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
5048                 CHECK(t->tables, ENOMEM);
5049
5050                 TAILQ_FOREACH(table, &p->tables, node) {
5051                         struct table_runtime *r = &t->tables[table->id];
5052
5053                         if (table->type) {
5054                                 uint64_t size;
5055
5056                                 size = table->type->ops.mailbox_size_get();
5057
5058                                 /* r->func. */
5059                                 r->func = table->type->ops.lkp;
5060
5061                                 /* r->mailbox. */
5062                                 if (size) {
5063                                         r->mailbox = calloc(1, size);
5064                                         CHECK(r->mailbox, ENOMEM);
5065                                 }
5066
5067                                 /* r->key. */
5068                                 r->key = table->is_header ?
5069                                         &t->structs[table->header->struct_id] :
5070                                         &t->structs[p->metadata_struct_id];
5071                         } else {
5072                                 r->func = table_stub_lkp;
5073                         }
5074                 }
5075         }
5076
5077         return 0;
5078 }
5079
5080 static void
5081 table_build_free(struct rte_swx_pipeline *p)
5082 {
5083         uint32_t i;
5084
5085         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
5086                 struct thread *t = &p->threads[i];
5087                 uint32_t j;
5088
5089                 if (!t->tables)
5090                         continue;
5091
5092                 for (j = 0; j < p->n_tables; j++) {
5093                         struct table_runtime *r = &t->tables[j];
5094
5095                         free(r->mailbox);
5096                 }
5097
5098                 free(t->tables);
5099                 t->tables = NULL;
5100         }
5101 }
5102
5103 static void
5104 table_free(struct rte_swx_pipeline *p)
5105 {
5106         table_build_free(p);
5107
5108         /* Tables. */
5109         for ( ; ; ) {
5110                 struct table *elem;
5111
5112                 elem = TAILQ_FIRST(&p->tables);
5113                 if (!elem)
5114                         break;
5115
5116                 TAILQ_REMOVE(&p->tables, elem, node);
5117                 free(elem->fields);
5118                 free(elem->actions);
5119                 free(elem->default_action_data);
5120                 free(elem);
5121         }
5122
5123         /* Table types. */
5124         for ( ; ; ) {
5125                 struct table_type *elem;
5126
5127                 elem = TAILQ_FIRST(&p->table_types);
5128                 if (!elem)
5129                         break;
5130
5131                 TAILQ_REMOVE(&p->table_types, elem, node);
5132                 free(elem);
5133         }
5134 }
5135
5136 /*
5137  * Pipeline.
5138  */
5139 int
5140 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
5141 {
5142         struct rte_swx_pipeline *pipeline;
5143
5144         /* Check input parameters. */
5145         CHECK(p, EINVAL);
5146
5147         /* Memory allocation. */
5148         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
5149         CHECK(pipeline, ENOMEM);
5150
5151         /* Initialization. */
5152         TAILQ_INIT(&pipeline->struct_types);
5153         TAILQ_INIT(&pipeline->port_in_types);
5154         TAILQ_INIT(&pipeline->ports_in);
5155         TAILQ_INIT(&pipeline->port_out_types);
5156         TAILQ_INIT(&pipeline->ports_out);
5157         TAILQ_INIT(&pipeline->extern_types);
5158         TAILQ_INIT(&pipeline->extern_objs);
5159         TAILQ_INIT(&pipeline->extern_funcs);
5160         TAILQ_INIT(&pipeline->headers);
5161         TAILQ_INIT(&pipeline->actions);
5162         TAILQ_INIT(&pipeline->table_types);
5163         TAILQ_INIT(&pipeline->tables);
5164
5165         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
5166         pipeline->numa_node = numa_node;
5167
5168         *p = pipeline;
5169         return 0;
5170 }
5171
5172 void
5173 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
5174 {
5175         if (!p)
5176                 return;
5177
5178         free(p->instructions);
5179
5180         table_state_free(p);
5181         table_free(p);
5182         action_free(p);
5183         metadata_free(p);
5184         header_free(p);
5185         extern_func_free(p);
5186         extern_obj_free(p);
5187         port_out_free(p);
5188         port_in_free(p);
5189         struct_free(p);
5190
5191         free(p);
5192 }
5193
5194 int
5195 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
5196                                      const char **instructions,
5197                                      uint32_t n_instructions)
5198 {
5199         int err;
5200         uint32_t i;
5201
5202         err = instruction_config(p, NULL, instructions, n_instructions);
5203         if (err)
5204                 return err;
5205
5206         /* Thread instruction pointer reset. */
5207         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
5208                 struct thread *t = &p->threads[i];
5209
5210                 thread_ip_reset(p, t);
5211         }
5212
5213         return 0;
5214 }
5215
5216 int
5217 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
5218 {
5219         int status;
5220
5221         CHECK(p, EINVAL);
5222         CHECK(p->build_done == 0, EEXIST);
5223
5224         status = port_in_build(p);
5225         if (status)
5226                 goto error;
5227
5228         status = port_out_build(p);
5229         if (status)
5230                 goto error;
5231
5232         status = struct_build(p);
5233         if (status)
5234                 goto error;
5235
5236         status = extern_obj_build(p);
5237         if (status)
5238                 goto error;
5239
5240         status = extern_func_build(p);
5241         if (status)
5242                 goto error;
5243
5244         status = header_build(p);
5245         if (status)
5246                 goto error;
5247
5248         status = metadata_build(p);
5249         if (status)
5250                 goto error;
5251
5252         status = action_build(p);
5253         if (status)
5254                 goto error;
5255
5256         status = table_build(p);
5257         if (status)
5258                 goto error;
5259
5260         status = table_state_build(p);
5261         if (status)
5262                 goto error;
5263
5264         p->build_done = 1;
5265         return 0;
5266
5267 error:
5268         table_state_build_free(p);
5269         table_build_free(p);
5270         action_build_free(p);
5271         metadata_build_free(p);
5272         header_build_free(p);
5273         extern_func_build_free(p);
5274         extern_obj_build_free(p);
5275         port_out_build_free(p);
5276         port_in_build_free(p);
5277         struct_build_free(p);
5278
5279         return status;
5280 }
5281
5282 void
5283 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
5284 {
5285         uint32_t i;
5286
5287         for (i = 0; i < n_instructions; i++)
5288                 instr_exec(p);
5289 }
5290
5291 /*
5292  * Control.
5293  */
5294 int
5295 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
5296                                  struct rte_swx_table_state **table_state)
5297 {
5298         if (!p || !table_state || !p->build_done)
5299                 return -EINVAL;
5300
5301         *table_state = p->table_state;
5302         return 0;
5303 }
5304
5305 int
5306 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
5307                                  struct rte_swx_table_state *table_state)
5308 {
5309         if (!p || !table_state || !p->build_done)
5310                 return -EINVAL;
5311
5312         p->table_state = table_state;
5313         return 0;
5314 }