lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9 #include <arpa/inet.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_prefetch.h>
  13 #include <rte_byteorder.h>
  14
  15 #include "rte_swx_pipeline.h"
  16 #include "rte_swx_ctl.h"
  17
  18 #define CHECK(condition, err_code)                                             \
  19 do {                                                                           \
  20         if (!(condition))                                                      \
  21                 return -(err_code);                                            \
  22 } while (0)
  23
  24 #define CHECK_NAME(name, err_code)                                             \
  25         CHECK((name) && (name)[0], err_code)
  26
  27 #ifndef TRACE_LEVEL
  28 #define TRACE_LEVEL 0
  29 #endif
  30
  31 #if TRACE_LEVEL
  32 #define TRACE(...) printf(__VA_ARGS__)
  33 #else
  34 #define TRACE(...)
  35 #endif
  36
  37 #define ntoh64(x) rte_be_to_cpu_64(x)
  38 #define hton64(x) rte_cpu_to_be_64(x)
  39
  40 /*
  41  * Struct.
  42  */
  43 struct field {
  44         char name[RTE_SWX_NAME_SIZE];
  45         uint32_t n_bits;
  46         uint32_t offset;
  47 };
  48
  49 struct struct_type {
  50         TAILQ_ENTRY(struct_type) node;
  51         char name[RTE_SWX_NAME_SIZE];
  52         struct field *fields;
  53         uint32_t n_fields;
  54         uint32_t n_bits;
  55 };
  56
  57 TAILQ_HEAD(struct_type_tailq, struct_type);
  58
  59 /*
  60  * Input port.
  61  */
  62 struct port_in_type {
  63         TAILQ_ENTRY(port_in_type) node;
  64         char name[RTE_SWX_NAME_SIZE];
  65         struct rte_swx_port_in_ops ops;
  66 };
  67
  68 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  69
  70 struct port_in {
  71         TAILQ_ENTRY(port_in) node;
  72         struct port_in_type *type;
  73         void *obj;
  74         uint32_t id;
  75 };
  76
  77 TAILQ_HEAD(port_in_tailq, port_in);
  78
  79 struct port_in_runtime {
  80         rte_swx_port_in_pkt_rx_t pkt_rx;
  81         void *obj;
  82 };
  83
  84 /*
  85  * Output port.
  86  */
  87 struct port_out_type {
  88         TAILQ_ENTRY(port_out_type) node;
  89         char name[RTE_SWX_NAME_SIZE];
  90         struct rte_swx_port_out_ops ops;
  91 };
  92
  93 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  94
  95 struct port_out {
  96         TAILQ_ENTRY(port_out) node;
  97         struct port_out_type *type;
  98         void *obj;
  99         uint32_t id;
 100 };
 101
 102 TAILQ_HEAD(port_out_tailq, port_out);
 103
 104 struct port_out_runtime {
 105         rte_swx_port_out_pkt_tx_t pkt_tx;
 106         rte_swx_port_out_flush_t flush;
 107         void *obj;
 108 };
 109
 110 /*
 111  * Extern object.
 112  */
 113 struct extern_type_member_func {
 114         TAILQ_ENTRY(extern_type_member_func) node;
 115         char name[RTE_SWX_NAME_SIZE];
 116         rte_swx_extern_type_member_func_t func;
 117         uint32_t id;
 118 };
 119
 120 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 121
 122 struct extern_type {
 123         TAILQ_ENTRY(extern_type) node;
 124         char name[RTE_SWX_NAME_SIZE];
 125         struct struct_type *mailbox_struct_type;
 126         rte_swx_extern_type_constructor_t constructor;
 127         rte_swx_extern_type_destructor_t destructor;
 128         struct extern_type_member_func_tailq funcs;
 129         uint32_t n_funcs;
 130 };
 131
 132 TAILQ_HEAD(extern_type_tailq, extern_type);
 133
 134 struct extern_obj {
 135         TAILQ_ENTRY(extern_obj) node;
 136         char name[RTE_SWX_NAME_SIZE];
 137         struct extern_type *type;
 138         void *obj;
 139         uint32_t struct_id;
 140         uint32_t id;
 141 };
 142
 143 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 144
 145 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 146 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 147 #endif
 148
 149 struct extern_obj_runtime {
 150         void *obj;
 151         uint8_t *mailbox;
 152         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 153 };
 154
 155 /*
 156  * Extern function.
 157  */
 158 struct extern_func {
 159         TAILQ_ENTRY(extern_func) node;
 160         char name[RTE_SWX_NAME_SIZE];
 161         struct struct_type *mailbox_struct_type;
 162         rte_swx_extern_func_t func;
 163         uint32_t struct_id;
 164         uint32_t id;
 165 };
 166
 167 TAILQ_HEAD(extern_func_tailq, extern_func);
 168
 169 struct extern_func_runtime {
 170         uint8_t *mailbox;
 171         rte_swx_extern_func_t func;
 172 };
 173
 174 /*
 175  * Header.
 176  */
 177 struct header {
 178         TAILQ_ENTRY(header) node;
 179         char name[RTE_SWX_NAME_SIZE];
 180         struct struct_type *st;
 181         uint32_t struct_id;
 182         uint32_t id;
 183 };
 184
 185 TAILQ_HEAD(header_tailq, header);
 186
 187 struct header_runtime {
 188         uint8_t *ptr0;
 189 };
 190
 191 struct header_out_runtime {
 192         uint8_t *ptr0;
 193         uint8_t *ptr;
 194         uint32_t n_bytes;
 195 };
 196
 197 /*
 198  * Instruction.
 199  */
 200
 201 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 202  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 203  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 204  * when transferred to packet meta-data and in NBO when transferred to packet
 205  * headers.
 206  */
 207
 208 /* Notation conventions:
 209  *    -Header field: H = h.header.field (dst/src)
 210  *    -Meta-data field: M = m.field (dst/src)
 211  *    -Extern object mailbox field: E = e.field (dst/src)
 212  *    -Extern function mailbox field: F = f.field (dst/src)
 213  *    -Table action data field: T = t.field (src only)
 214  *    -Immediate value: I = 32-bit unsigned value (src only)
 215  */
 216
 217 enum instruction_type {
 218         /* rx m.port_in */
 219         INSTR_RX,
 220
 221         /* tx m.port_out */
 222         INSTR_TX,
 223
 224         /* extract h.header */
 225         INSTR_HDR_EXTRACT,
 226         INSTR_HDR_EXTRACT2,
 227         INSTR_HDR_EXTRACT3,
 228         INSTR_HDR_EXTRACT4,
 229         INSTR_HDR_EXTRACT5,
 230         INSTR_HDR_EXTRACT6,
 231         INSTR_HDR_EXTRACT7,
 232         INSTR_HDR_EXTRACT8,
 233
 234         /* emit h.header */
 235         INSTR_HDR_EMIT,
 236         INSTR_HDR_EMIT_TX,
 237         INSTR_HDR_EMIT2_TX,
 238         INSTR_HDR_EMIT3_TX,
 239         INSTR_HDR_EMIT4_TX,
 240         INSTR_HDR_EMIT5_TX,
 241         INSTR_HDR_EMIT6_TX,
 242         INSTR_HDR_EMIT7_TX,
 243         INSTR_HDR_EMIT8_TX,
 244
 245         /* validate h.header */
 246         INSTR_HDR_VALIDATE,
 247
 248         /* invalidate h.header */
 249         INSTR_HDR_INVALIDATE,
 250
 251         /* mov dst src
 252          * dst = src
 253          * dst = HMEF, src = HMEFTI
 254          */
 255         INSTR_MOV,   /* dst = MEF, src = MEFT */
 256         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 257         INSTR_MOV_I, /* dst = HMEF, src = I */
 258
 259         /* dma h.header t.field
 260          * memcpy(h.header, t.field, sizeof(h.header))
 261          */
 262         INSTR_DMA_HT,
 263         INSTR_DMA_HT2,
 264         INSTR_DMA_HT3,
 265         INSTR_DMA_HT4,
 266         INSTR_DMA_HT5,
 267         INSTR_DMA_HT6,
 268         INSTR_DMA_HT7,
 269         INSTR_DMA_HT8,
 270
 271         /* add dst src
 272          * dst += src
 273          * dst = HMEF, src = HMEFTI
 274          */
 275         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 276         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 277         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 278         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 279         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 280         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 281
 282         /* sub dst src
 283          * dst -= src
 284          * dst = HMEF, src = HMEFTI
 285          */
 286         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 287         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 288         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 289         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 290         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 291         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 292
 293         /* ckadd dst src
 294          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 295          * dst = H, src = {H, h.header}
 296          */
 297         INSTR_ALU_CKADD_FIELD,    /* src = H */
 298         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 299         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 300
 301         /* cksub dst src
 302          * dst = dst '- src
 303          * dst = H, src = H
 304          */
 305         INSTR_ALU_CKSUB_FIELD,
 306
 307         /* and dst src
 308          * dst &= src
 309          * dst = HMEF, src = HMEFTI
 310          */
 311         INSTR_ALU_AND,   /* dst = MEF, src = MEFT */
 312         INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 313         INSTR_ALU_AND_I, /* dst = HMEF, src = I */
 314
 315         /* or dst src
 316          * dst |= src
 317          * dst = HMEF, src = HMEFTI
 318          */
 319         INSTR_ALU_OR,   /* dst = MEF, src = MEFT */
 320         INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 321         INSTR_ALU_OR_I, /* dst = HMEF, src = I */
 322
 323         /* xor dst src
 324          * dst ^= src
 325          * dst = HMEF, src = HMEFTI
 326          */
 327         INSTR_ALU_XOR,   /* dst = MEF, src = MEFT */
 328         INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 329         INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
 330 };
 331
 332 struct instr_operand {
 333         uint8_t struct_id;
 334         uint8_t n_bits;
 335         uint8_t offset;
 336         uint8_t pad;
 337 };
 338
 339 struct instr_io {
 340         struct {
 341                 uint8_t offset;
 342                 uint8_t n_bits;
 343                 uint8_t pad[2];
 344         } io;
 345
 346         struct {
 347                 uint8_t header_id[8];
 348                 uint8_t struct_id[8];
 349                 uint8_t n_bytes[8];
 350         } hdr;
 351 };
 352
 353 struct instr_hdr_validity {
 354         uint8_t header_id;
 355 };
 356
 357 struct instr_dst_src {
 358         struct instr_operand dst;
 359         union {
 360                 struct instr_operand src;
 361                 uint32_t src_val;
 362         };
 363 };
 364
 365 struct instr_dma {
 366         struct {
 367                 uint8_t header_id[8];
 368                 uint8_t struct_id[8];
 369         } dst;
 370
 371         struct {
 372                 uint8_t offset[8];
 373         } src;
 374
 375         uint16_t n_bytes[8];
 376 };
 377
 378 struct instruction {
 379         enum instruction_type type;
 380         union {
 381                 struct instr_io io;
 382                 struct instr_hdr_validity valid;
 383                 struct instr_dst_src mov;
 384                 struct instr_dma dma;
 385                 struct instr_dst_src alu;
 386         };
 387 };
 388
 389 struct instruction_data {
 390         char label[RTE_SWX_NAME_SIZE];
 391         char jmp_label[RTE_SWX_NAME_SIZE];
 392         uint32_t n_users; /* user = jmp instruction to this instruction. */
 393         int invalid;
 394 };
 395
 396 /*
 397  * Action.
 398  */
 399 struct action {
 400         TAILQ_ENTRY(action) node;
 401         char name[RTE_SWX_NAME_SIZE];
 402         struct struct_type *st;
 403         struct instruction *instructions;
 404         uint32_t n_instructions;
 405         uint32_t id;
 406 };
 407
 408 TAILQ_HEAD(action_tailq, action);
 409
 410 /*
 411  * Table.
 412  */
 413 struct table_type {
 414         TAILQ_ENTRY(table_type) node;
 415         char name[RTE_SWX_NAME_SIZE];
 416         enum rte_swx_table_match_type match_type;
 417         struct rte_swx_table_ops ops;
 418 };
 419
 420 TAILQ_HEAD(table_type_tailq, table_type);
 421
 422 struct match_field {
 423         enum rte_swx_table_match_type match_type;
 424         struct field *field;
 425 };
 426
 427 struct table {
 428         TAILQ_ENTRY(table) node;
 429         char name[RTE_SWX_NAME_SIZE];
 430         char args[RTE_SWX_NAME_SIZE];
 431         struct table_type *type; /* NULL when n_fields == 0. */
 432
 433         /* Match. */
 434         struct match_field *fields;
 435         uint32_t n_fields;
 436         int is_header; /* Only valid when n_fields > 0. */
 437         struct header *header; /* Only valid when n_fields > 0. */
 438
 439         /* Action. */
 440         struct action **actions;
 441         struct action *default_action;
 442         uint8_t *default_action_data;
 443         uint32_t n_actions;
 444         int default_action_is_const;
 445         uint32_t action_data_size_max;
 446
 447         uint32_t size;
 448         uint32_t id;
 449 };
 450
 451 TAILQ_HEAD(table_tailq, table);
 452
 453 struct table_runtime {
 454         rte_swx_table_lookup_t func;
 455         void *mailbox;
 456         uint8_t **key;
 457 };
 458
 459 /*
 460  * Pipeline.
 461  */
 462 struct thread {
 463         /* Packet. */
 464         struct rte_swx_pkt pkt;
 465         uint8_t *ptr;
 466
 467         /* Structures. */
 468         uint8_t **structs;
 469
 470         /* Packet headers. */
 471         struct header_runtime *headers; /* Extracted or generated headers. */
 472         struct header_out_runtime *headers_out; /* Emitted headers. */
 473         uint8_t *header_storage;
 474         uint8_t *header_out_storage;
 475         uint64_t valid_headers;
 476         uint32_t n_headers_out;
 477
 478         /* Packet meta-data. */
 479         uint8_t *metadata;
 480
 481         /* Tables. */
 482         struct table_runtime *tables;
 483         struct rte_swx_table_state *table_state;
 484         uint64_t action_id;
 485         int hit; /* 0 = Miss, 1 = Hit. */
 486
 487         /* Extern objects and functions. */
 488         struct extern_obj_runtime *extern_objs;
 489         struct extern_func_runtime *extern_funcs;
 490
 491         /* Instructions. */
 492         struct instruction *ip;
 493         struct instruction *ret;
 494 };
 495
 496 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 497 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 498 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 499
 500 #define ALU(thread, ip, operator)  \
 501 {                                                                              \
 502         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 503         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 504         uint64_t dst64 = *dst64_ptr;                                           \
 505         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 506         uint64_t dst = dst64 & dst64_mask;                                     \
 507                                                                                \
 508         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 509         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 510         uint64_t src64 = *src64_ptr;                                           \
 511         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 512         uint64_t src = src64 & src64_mask;                                     \
 513                                                                                \
 514         uint64_t result = dst operator src;                                    \
 515                                                                                \
 516         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 517 }
 518
 519 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 520
 521 #define ALU_S(thread, ip, operator)  \
 522 {                                                                              \
 523         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 524         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 525         uint64_t dst64 = *dst64_ptr;                                           \
 526         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 527         uint64_t dst = dst64 & dst64_mask;                                     \
 528                                                                                \
 529         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 530         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 531         uint64_t src64 = *src64_ptr;                                           \
 532         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 533                                                                                \
 534         uint64_t result = dst operator src;                                    \
 535                                                                                \
 536         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 537 }
 538
 539 #define ALU_MH ALU_S
 540
 541 #define ALU_HM(thread, ip, operator)  \
 542 {                                                                              \
 543         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 544         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 545         uint64_t dst64 = *dst64_ptr;                                           \
 546         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 547         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 548                                                                                \
 549         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 550         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 551         uint64_t src64 = *src64_ptr;                                           \
 552         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 553         uint64_t src = src64 & src64_mask;                                     \
 554                                                                                \
 555         uint64_t result = dst operator src;                                    \
 556         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 557                                                                                \
 558         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 559 }
 560
 561 #define ALU_HH(thread, ip, operator)  \
 562 {                                                                              \
 563         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 564         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 565         uint64_t dst64 = *dst64_ptr;                                           \
 566         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 567         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 568                                                                                \
 569         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 570         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 571         uint64_t src64 = *src64_ptr;                                           \
 572         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 573                                                                                \
 574         uint64_t result = dst operator src;                                    \
 575         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 576                                                                                \
 577         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 578 }
 579
 580 #else
 581
 582 #define ALU_S ALU
 583 #define ALU_MH ALU
 584 #define ALU_HM ALU
 585 #define ALU_HH ALU
 586
 587 #endif
 588
 589 #define ALU_I(thread, ip, operator)  \
 590 {                                                                              \
 591         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 592         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 593         uint64_t dst64 = *dst64_ptr;                                           \
 594         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 595         uint64_t dst = dst64 & dst64_mask;                                     \
 596                                                                                \
 597         uint64_t src = (ip)->alu.src_val;                                      \
 598                                                                                \
 599         uint64_t result = dst operator src;                                    \
 600                                                                                \
 601         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 602 }
 603
 604 #define ALU_MI ALU_I
 605
 606 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 607
 608 #define ALU_HI(thread, ip, operator)  \
 609 {                                                                              \
 610         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 611         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 612         uint64_t dst64 = *dst64_ptr;                                           \
 613         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 614         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 615                                                                                \
 616         uint64_t src = (ip)->alu.src_val;                                      \
 617                                                                                \
 618         uint64_t result = dst operator src;                                    \
 619         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 620                                                                                \
 621         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 622 }
 623
 624 #else
 625
 626 #define ALU_HI ALU_I
 627
 628 #endif
 629
 630 #define MOV(thread, ip)  \
 631 {                                                                              \
 632         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 633         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 634         uint64_t dst64 = *dst64_ptr;                                           \
 635         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 636                                                                                \
 637         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 638         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 639         uint64_t src64 = *src64_ptr;                                           \
 640         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
 641         uint64_t src = src64 & src64_mask;                                     \
 642                                                                                \
 643         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 644 }
 645
 646 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 647
 648 #define MOV_S(thread, ip)  \
 649 {                                                                              \
 650         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 651         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 652         uint64_t dst64 = *dst64_ptr;                                           \
 653         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 654                                                                                \
 655         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 656         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 657         uint64_t src64 = *src64_ptr;                                           \
 658         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
 659                                                                                \
 660         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 661 }
 662
 663 #else
 664
 665 #define MOV_S MOV
 666
 667 #endif
 668
 669 #define MOV_I(thread, ip)  \
 670 {                                                                              \
 671         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 672         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 673         uint64_t dst64 = *dst64_ptr;                                           \
 674         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 675                                                                                \
 676         uint64_t src = (ip)->mov.src_val;                                      \
 677                                                                                \
 678         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 679 }
 680
 681 #define METADATA_READ(thread, offset, n_bits)                                  \
 682 ({                                                                             \
 683         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 684         uint64_t m64 = *m64_ptr;                                               \
 685         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 686         (m64 & m64_mask);                                                      \
 687 })
 688
 689 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
 690 {                                                                              \
 691         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 692         uint64_t m64 = *m64_ptr;                                               \
 693         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 694                                                                                \
 695         uint64_t m_new = value;                                                \
 696                                                                                \
 697         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
 698 }
 699
 700 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 701 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 702 #endif
 703
 704 struct rte_swx_pipeline {
 705         struct struct_type_tailq struct_types;
 706         struct port_in_type_tailq port_in_types;
 707         struct port_in_tailq ports_in;
 708         struct port_out_type_tailq port_out_types;
 709         struct port_out_tailq ports_out;
 710         struct extern_type_tailq extern_types;
 711         struct extern_obj_tailq extern_objs;
 712         struct extern_func_tailq extern_funcs;
 713         struct header_tailq headers;
 714         struct struct_type *metadata_st;
 715         uint32_t metadata_struct_id;
 716         struct action_tailq actions;
 717         struct table_type_tailq table_types;
 718         struct table_tailq tables;
 719
 720         struct port_in_runtime *in;
 721         struct port_out_runtime *out;
 722         struct instruction **action_instructions;
 723         struct rte_swx_table_state *table_state;
 724         struct instruction *instructions;
 725         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 726
 727         uint32_t n_structs;
 728         uint32_t n_ports_in;
 729         uint32_t n_ports_out;
 730         uint32_t n_extern_objs;
 731         uint32_t n_extern_funcs;
 732         uint32_t n_actions;
 733         uint32_t n_tables;
 734         uint32_t n_headers;
 735         uint32_t thread_id;
 736         uint32_t port_id;
 737         uint32_t n_instructions;
 738         int build_done;
 739         int numa_node;
 740 };
 741
 742 /*
 743  * Struct.
 744  */
 745 static struct struct_type *
 746 struct_type_find(struct rte_swx_pipeline *p, const char *name)
 747 {
 748         struct struct_type *elem;
 749
 750         TAILQ_FOREACH(elem, &p->struct_types, node)
 751                 if (strcmp(elem->name, name) == 0)
 752                         return elem;
 753
 754         return NULL;
 755 }
 756
 757 static struct field *
 758 struct_type_field_find(struct struct_type *st, const char *name)
 759 {
 760         uint32_t i;
 761
 762         for (i = 0; i < st->n_fields; i++) {
 763                 struct field *f = &st->fields[i];
 764
 765                 if (strcmp(f->name, name) == 0)
 766                         return f;
 767         }
 768
 769         return NULL;
 770 }
 771
 772 int
 773 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 774                                       const char *name,
 775                                       struct rte_swx_field_params *fields,
 776                                       uint32_t n_fields)
 777 {
 778         struct struct_type *st;
 779         uint32_t i;
 780
 781         CHECK(p, EINVAL);
 782         CHECK_NAME(name, EINVAL);
 783         CHECK(fields, EINVAL);
 784         CHECK(n_fields, EINVAL);
 785
 786         for (i = 0; i < n_fields; i++) {
 787                 struct rte_swx_field_params *f = &fields[i];
 788                 uint32_t j;
 789
 790                 CHECK_NAME(f->name, EINVAL);
 791                 CHECK(f->n_bits, EINVAL);
 792                 CHECK(f->n_bits <= 64, EINVAL);
 793                 CHECK((f->n_bits & 7) == 0, EINVAL);
 794
 795                 for (j = 0; j < i; j++) {
 796                         struct rte_swx_field_params *f_prev = &fields[j];
 797
 798                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 799                 }
 800         }
 801
 802         CHECK(!struct_type_find(p, name), EEXIST);
 803
 804         /* Node allocation. */
 805         st = calloc(1, sizeof(struct struct_type));
 806         CHECK(st, ENOMEM);
 807
 808         st->fields = calloc(n_fields, sizeof(struct field));
 809         if (!st->fields) {
 810                 free(st);
 811                 CHECK(0, ENOMEM);
 812         }
 813
 814         /* Node initialization. */
 815         strcpy(st->name, name);
 816         for (i = 0; i < n_fields; i++) {
 817                 struct field *dst = &st->fields[i];
 818                 struct rte_swx_field_params *src = &fields[i];
 819
 820                 strcpy(dst->name, src->name);
 821                 dst->n_bits = src->n_bits;
 822                 dst->offset = st->n_bits;
 823
 824                 st->n_bits += src->n_bits;
 825         }
 826         st->n_fields = n_fields;
 827
 828         /* Node add to tailq. */
 829         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 830
 831         return 0;
 832 }
 833
 834 static int
 835 struct_build(struct rte_swx_pipeline *p)
 836 {
 837         uint32_t i;
 838
 839         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 840                 struct thread *t = &p->threads[i];
 841
 842                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 843                 CHECK(t->structs, ENOMEM);
 844         }
 845
 846         return 0;
 847 }
 848
 849 static void
 850 struct_build_free(struct rte_swx_pipeline *p)
 851 {
 852         uint32_t i;
 853
 854         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 855                 struct thread *t = &p->threads[i];
 856
 857                 free(t->structs);
 858                 t->structs = NULL;
 859         }
 860 }
 861
 862 static void
 863 struct_free(struct rte_swx_pipeline *p)
 864 {
 865         struct_build_free(p);
 866
 867         /* Struct types. */
 868         for ( ; ; ) {
 869                 struct struct_type *elem;
 870
 871                 elem = TAILQ_FIRST(&p->struct_types);
 872                 if (!elem)
 873                         break;
 874
 875                 TAILQ_REMOVE(&p->struct_types, elem, node);
 876                 free(elem->fields);
 877                 free(elem);
 878         }
 879 }
 880
 881 /*
 882  * Input port.
 883  */
 884 static struct port_in_type *
 885 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 886 {
 887         struct port_in_type *elem;
 888
 889         if (!name)
 890                 return NULL;
 891
 892         TAILQ_FOREACH(elem, &p->port_in_types, node)
 893                 if (strcmp(elem->name, name) == 0)
 894                         return elem;
 895
 896         return NULL;
 897 }
 898
 899 int
 900 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 901                                        const char *name,
 902                                        struct rte_swx_port_in_ops *ops)
 903 {
 904         struct port_in_type *elem;
 905
 906         CHECK(p, EINVAL);
 907         CHECK_NAME(name, EINVAL);
 908         CHECK(ops, EINVAL);
 909         CHECK(ops->create, EINVAL);
 910         CHECK(ops->free, EINVAL);
 911         CHECK(ops->pkt_rx, EINVAL);
 912         CHECK(ops->stats_read, EINVAL);
 913
 914         CHECK(!port_in_type_find(p, name), EEXIST);
 915
 916         /* Node allocation. */
 917         elem = calloc(1, sizeof(struct port_in_type));
 918         CHECK(elem, ENOMEM);
 919
 920         /* Node initialization. */
 921         strcpy(elem->name, name);
 922         memcpy(&elem->ops, ops, sizeof(*ops));
 923
 924         /* Node add to tailq. */
 925         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 926
 927         return 0;
 928 }
 929
 930 static struct port_in *
 931 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 932 {
 933         struct port_in *port;
 934
 935         TAILQ_FOREACH(port, &p->ports_in, node)
 936                 if (port->id == port_id)
 937                         return port;
 938
 939         return NULL;
 940 }
 941
 942 int
 943 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 944                                 uint32_t port_id,
 945                                 const char *port_type_name,
 946                                 void *args)
 947 {
 948         struct port_in_type *type = NULL;
 949         struct port_in *port = NULL;
 950         void *obj = NULL;
 951
 952         CHECK(p, EINVAL);
 953
 954         CHECK(!port_in_find(p, port_id), EINVAL);
 955
 956         CHECK_NAME(port_type_name, EINVAL);
 957         type = port_in_type_find(p, port_type_name);
 958         CHECK(type, EINVAL);
 959
 960         obj = type->ops.create(args);
 961         CHECK(obj, ENODEV);
 962
 963         /* Node allocation. */
 964         port = calloc(1, sizeof(struct port_in));
 965         CHECK(port, ENOMEM);
 966
 967         /* Node initialization. */
 968         port->type = type;
 969         port->obj = obj;
 970         port->id = port_id;
 971
 972         /* Node add to tailq. */
 973         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 974         if (p->n_ports_in < port_id + 1)
 975                 p->n_ports_in = port_id + 1;
 976
 977         return 0;
 978 }
 979
 980 static int
 981 port_in_build(struct rte_swx_pipeline *p)
 982 {
 983         struct port_in *port;
 984         uint32_t i;
 985
 986         CHECK(p->n_ports_in, EINVAL);
 987         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 988
 989         for (i = 0; i < p->n_ports_in; i++)
 990                 CHECK(port_in_find(p, i), EINVAL);
 991
 992         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 993         CHECK(p->in, ENOMEM);
 994
 995         TAILQ_FOREACH(port, &p->ports_in, node) {
 996                 struct port_in_runtime *in = &p->in[port->id];
 997
 998                 in->pkt_rx = port->type->ops.pkt_rx;
 999                 in->obj = port->obj;
1000         }
1001
1002         return 0;
1003 }
1004
1005 static void
1006 port_in_build_free(struct rte_swx_pipeline *p)
1007 {
1008         free(p->in);
1009         p->in = NULL;
1010 }
1011
1012 static void
1013 port_in_free(struct rte_swx_pipeline *p)
1014 {
1015         port_in_build_free(p);
1016
1017         /* Input ports. */
1018         for ( ; ; ) {
1019                 struct port_in *port;
1020
1021                 port = TAILQ_FIRST(&p->ports_in);
1022                 if (!port)
1023                         break;
1024
1025                 TAILQ_REMOVE(&p->ports_in, port, node);
1026                 port->type->ops.free(port->obj);
1027                 free(port);
1028         }
1029
1030         /* Input port types. */
1031         for ( ; ; ) {
1032                 struct port_in_type *elem;
1033
1034                 elem = TAILQ_FIRST(&p->port_in_types);
1035                 if (!elem)
1036                         break;
1037
1038                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1039                 free(elem);
1040         }
1041 }
1042
1043 /*
1044  * Output port.
1045  */
1046 static struct port_out_type *
1047 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1048 {
1049         struct port_out_type *elem;
1050
1051         if (!name)
1052                 return NULL;
1053
1054         TAILQ_FOREACH(elem, &p->port_out_types, node)
1055                 if (!strcmp(elem->name, name))
1056                         return elem;
1057
1058         return NULL;
1059 }
1060
1061 int
1062 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1063                                         const char *name,
1064                                         struct rte_swx_port_out_ops *ops)
1065 {
1066         struct port_out_type *elem;
1067
1068         CHECK(p, EINVAL);
1069         CHECK_NAME(name, EINVAL);
1070         CHECK(ops, EINVAL);
1071         CHECK(ops->create, EINVAL);
1072         CHECK(ops->free, EINVAL);
1073         CHECK(ops->pkt_tx, EINVAL);
1074         CHECK(ops->stats_read, EINVAL);
1075
1076         CHECK(!port_out_type_find(p, name), EEXIST);
1077
1078         /* Node allocation. */
1079         elem = calloc(1, sizeof(struct port_out_type));
1080         CHECK(elem, ENOMEM);
1081
1082         /* Node initialization. */
1083         strcpy(elem->name, name);
1084         memcpy(&elem->ops, ops, sizeof(*ops));
1085
1086         /* Node add to tailq. */
1087         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1088
1089         return 0;
1090 }
1091
1092 static struct port_out *
1093 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1094 {
1095         struct port_out *port;
1096
1097         TAILQ_FOREACH(port, &p->ports_out, node)
1098                 if (port->id == port_id)
1099                         return port;
1100
1101         return NULL;
1102 }
1103
1104 int
1105 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1106                                  uint32_t port_id,
1107                                  const char *port_type_name,
1108                                  void *args)
1109 {
1110         struct port_out_type *type = NULL;
1111         struct port_out *port = NULL;
1112         void *obj = NULL;
1113
1114         CHECK(p, EINVAL);
1115
1116         CHECK(!port_out_find(p, port_id), EINVAL);
1117
1118         CHECK_NAME(port_type_name, EINVAL);
1119         type = port_out_type_find(p, port_type_name);
1120         CHECK(type, EINVAL);
1121
1122         obj = type->ops.create(args);
1123         CHECK(obj, ENODEV);
1124
1125         /* Node allocation. */
1126         port = calloc(1, sizeof(struct port_out));
1127         CHECK(port, ENOMEM);
1128
1129         /* Node initialization. */
1130         port->type = type;
1131         port->obj = obj;
1132         port->id = port_id;
1133
1134         /* Node add to tailq. */
1135         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1136         if (p->n_ports_out < port_id + 1)
1137                 p->n_ports_out = port_id + 1;
1138
1139         return 0;
1140 }
1141
1142 static int
1143 port_out_build(struct rte_swx_pipeline *p)
1144 {
1145         struct port_out *port;
1146         uint32_t i;
1147
1148         CHECK(p->n_ports_out, EINVAL);
1149
1150         for (i = 0; i < p->n_ports_out; i++)
1151                 CHECK(port_out_find(p, i), EINVAL);
1152
1153         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1154         CHECK(p->out, ENOMEM);
1155
1156         TAILQ_FOREACH(port, &p->ports_out, node) {
1157                 struct port_out_runtime *out = &p->out[port->id];
1158
1159                 out->pkt_tx = port->type->ops.pkt_tx;
1160                 out->flush = port->type->ops.flush;
1161                 out->obj = port->obj;
1162         }
1163
1164         return 0;
1165 }
1166
1167 static void
1168 port_out_build_free(struct rte_swx_pipeline *p)
1169 {
1170         free(p->out);
1171         p->out = NULL;
1172 }
1173
1174 static void
1175 port_out_free(struct rte_swx_pipeline *p)
1176 {
1177         port_out_build_free(p);
1178
1179         /* Output ports. */
1180         for ( ; ; ) {
1181                 struct port_out *port;
1182
1183                 port = TAILQ_FIRST(&p->ports_out);
1184                 if (!port)
1185                         break;
1186
1187                 TAILQ_REMOVE(&p->ports_out, port, node);
1188                 port->type->ops.free(port->obj);
1189                 free(port);
1190         }
1191
1192         /* Output port types. */
1193         for ( ; ; ) {
1194                 struct port_out_type *elem;
1195
1196                 elem = TAILQ_FIRST(&p->port_out_types);
1197                 if (!elem)
1198                         break;
1199
1200                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1201                 free(elem);
1202         }
1203 }
1204
1205 /*
1206  * Extern object.
1207  */
1208 static struct extern_type *
1209 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1210 {
1211         struct extern_type *elem;
1212
1213         TAILQ_FOREACH(elem, &p->extern_types, node)
1214                 if (strcmp(elem->name, name) == 0)
1215                         return elem;
1216
1217         return NULL;
1218 }
1219
1220 static struct extern_type_member_func *
1221 extern_type_member_func_find(struct extern_type *type, const char *name)
1222 {
1223         struct extern_type_member_func *elem;
1224
1225         TAILQ_FOREACH(elem, &type->funcs, node)
1226                 if (strcmp(elem->name, name) == 0)
1227                         return elem;
1228
1229         return NULL;
1230 }
1231
1232 static struct extern_obj *
1233 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1234 {
1235         struct extern_obj *elem;
1236
1237         TAILQ_FOREACH(elem, &p->extern_objs, node)
1238                 if (strcmp(elem->name, name) == 0)
1239                         return elem;
1240
1241         return NULL;
1242 }
1243
1244 static struct field *
1245 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1246                                const char *name,
1247                                struct extern_obj **object)
1248 {
1249         struct extern_obj *obj;
1250         struct field *f;
1251         char *obj_name, *field_name;
1252
1253         if ((name[0] != 'e') || (name[1] != '.'))
1254                 return NULL;
1255
1256         obj_name = strdup(&name[2]);
1257         if (!obj_name)
1258                 return NULL;
1259
1260         field_name = strchr(obj_name, '.');
1261         if (!field_name) {
1262                 free(obj_name);
1263                 return NULL;
1264         }
1265
1266         *field_name = 0;
1267         field_name++;
1268
1269         obj = extern_obj_find(p, obj_name);
1270         if (!obj) {
1271                 free(obj_name);
1272                 return NULL;
1273         }
1274
1275         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1276         if (!f) {
1277                 free(obj_name);
1278                 return NULL;
1279         }
1280
1281         if (object)
1282                 *object = obj;
1283
1284         free(obj_name);
1285         return f;
1286 }
1287
1288 int
1289 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1290         const char *name,
1291         const char *mailbox_struct_type_name,
1292         rte_swx_extern_type_constructor_t constructor,
1293         rte_swx_extern_type_destructor_t destructor)
1294 {
1295         struct extern_type *elem;
1296         struct struct_type *mailbox_struct_type;
1297
1298         CHECK(p, EINVAL);
1299
1300         CHECK_NAME(name, EINVAL);
1301         CHECK(!extern_type_find(p, name), EEXIST);
1302
1303         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1304         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1305         CHECK(mailbox_struct_type, EINVAL);
1306
1307         CHECK(constructor, EINVAL);
1308         CHECK(destructor, EINVAL);
1309
1310         /* Node allocation. */
1311         elem = calloc(1, sizeof(struct extern_type));
1312         CHECK(elem, ENOMEM);
1313
1314         /* Node initialization. */
1315         strcpy(elem->name, name);
1316         elem->mailbox_struct_type = mailbox_struct_type;
1317         elem->constructor = constructor;
1318         elem->destructor = destructor;
1319         TAILQ_INIT(&elem->funcs);
1320
1321         /* Node add to tailq. */
1322         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1323
1324         return 0;
1325 }
1326
1327 int
1328 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1329         const char *extern_type_name,
1330         const char *name,
1331         rte_swx_extern_type_member_func_t member_func)
1332 {
1333         struct extern_type *type;
1334         struct extern_type_member_func *type_member;
1335
1336         CHECK(p, EINVAL);
1337
1338         CHECK(extern_type_name, EINVAL);
1339         type = extern_type_find(p, extern_type_name);
1340         CHECK(type, EINVAL);
1341         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1342
1343         CHECK(name, EINVAL);
1344         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1345
1346         CHECK(member_func, EINVAL);
1347
1348         /* Node allocation. */
1349         type_member = calloc(1, sizeof(struct extern_type_member_func));
1350         CHECK(type_member, ENOMEM);
1351
1352         /* Node initialization. */
1353         strcpy(type_member->name, name);
1354         type_member->func = member_func;
1355         type_member->id = type->n_funcs;
1356
1357         /* Node add to tailq. */
1358         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1359         type->n_funcs++;
1360
1361         return 0;
1362 }
1363
1364 int
1365 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1366                                       const char *extern_type_name,
1367                                       const char *name,
1368                                       const char *args)
1369 {
1370         struct extern_type *type;
1371         struct extern_obj *obj;
1372         void *obj_handle;
1373
1374         CHECK(p, EINVAL);
1375
1376         CHECK_NAME(extern_type_name, EINVAL);
1377         type = extern_type_find(p, extern_type_name);
1378         CHECK(type, EINVAL);
1379
1380         CHECK_NAME(name, EINVAL);
1381         CHECK(!extern_obj_find(p, name), EEXIST);
1382
1383         /* Node allocation. */
1384         obj = calloc(1, sizeof(struct extern_obj));
1385         CHECK(obj, ENOMEM);
1386
1387         /* Object construction. */
1388         obj_handle = type->constructor(args);
1389         if (!obj_handle) {
1390                 free(obj);
1391                 CHECK(0, ENODEV);
1392         }
1393
1394         /* Node initialization. */
1395         strcpy(obj->name, name);
1396         obj->type = type;
1397         obj->obj = obj_handle;
1398         obj->struct_id = p->n_structs;
1399         obj->id = p->n_extern_objs;
1400
1401         /* Node add to tailq. */
1402         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1403         p->n_extern_objs++;
1404         p->n_structs++;
1405
1406         return 0;
1407 }
1408
1409 static int
1410 extern_obj_build(struct rte_swx_pipeline *p)
1411 {
1412         uint32_t i;
1413
1414         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1415                 struct thread *t = &p->threads[i];
1416                 struct extern_obj *obj;
1417
1418                 t->extern_objs = calloc(p->n_extern_objs,
1419                                         sizeof(struct extern_obj_runtime));
1420                 CHECK(t->extern_objs, ENOMEM);
1421
1422                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1423                         struct extern_obj_runtime *r =
1424                                 &t->extern_objs[obj->id];
1425                         struct extern_type_member_func *func;
1426                         uint32_t mailbox_size =
1427                                 obj->type->mailbox_struct_type->n_bits / 8;
1428
1429                         r->obj = obj->obj;
1430
1431                         r->mailbox = calloc(1, mailbox_size);
1432                         CHECK(r->mailbox, ENOMEM);
1433
1434                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1435                                 r->funcs[func->id] = func->func;
1436
1437                         t->structs[obj->struct_id] = r->mailbox;
1438                 }
1439         }
1440
1441         return 0;
1442 }
1443
1444 static void
1445 extern_obj_build_free(struct rte_swx_pipeline *p)
1446 {
1447         uint32_t i;
1448
1449         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1450                 struct thread *t = &p->threads[i];
1451                 uint32_t j;
1452
1453                 if (!t->extern_objs)
1454                         continue;
1455
1456                 for (j = 0; j < p->n_extern_objs; j++) {
1457                         struct extern_obj_runtime *r = &t->extern_objs[j];
1458
1459                         free(r->mailbox);
1460                 }
1461
1462                 free(t->extern_objs);
1463                 t->extern_objs = NULL;
1464         }
1465 }
1466
1467 static void
1468 extern_obj_free(struct rte_swx_pipeline *p)
1469 {
1470         extern_obj_build_free(p);
1471
1472         /* Extern objects. */
1473         for ( ; ; ) {
1474                 struct extern_obj *elem;
1475
1476                 elem = TAILQ_FIRST(&p->extern_objs);
1477                 if (!elem)
1478                         break;
1479
1480                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1481                 if (elem->obj)
1482                         elem->type->destructor(elem->obj);
1483                 free(elem);
1484         }
1485
1486         /* Extern types. */
1487         for ( ; ; ) {
1488                 struct extern_type *elem;
1489
1490                 elem = TAILQ_FIRST(&p->extern_types);
1491                 if (!elem)
1492                         break;
1493
1494                 TAILQ_REMOVE(&p->extern_types, elem, node);
1495
1496                 for ( ; ; ) {
1497                         struct extern_type_member_func *func;
1498
1499                         func = TAILQ_FIRST(&elem->funcs);
1500                         if (!func)
1501                                 break;
1502
1503                         TAILQ_REMOVE(&elem->funcs, func, node);
1504                         free(func);
1505                 }
1506
1507                 free(elem);
1508         }
1509 }
1510
1511 /*
1512  * Extern function.
1513  */
1514 static struct extern_func *
1515 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1516 {
1517         struct extern_func *elem;
1518
1519         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1520                 if (strcmp(elem->name, name) == 0)
1521                         return elem;
1522
1523         return NULL;
1524 }
1525
1526 static struct field *
1527 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1528                                 const char *name,
1529                                 struct extern_func **function)
1530 {
1531         struct extern_func *func;
1532         struct field *f;
1533         char *func_name, *field_name;
1534
1535         if ((name[0] != 'f') || (name[1] != '.'))
1536                 return NULL;
1537
1538         func_name = strdup(&name[2]);
1539         if (!func_name)
1540                 return NULL;
1541
1542         field_name = strchr(func_name, '.');
1543         if (!field_name) {
1544                 free(func_name);
1545                 return NULL;
1546         }
1547
1548         *field_name = 0;
1549         field_name++;
1550
1551         func = extern_func_find(p, func_name);
1552         if (!func) {
1553                 free(func_name);
1554                 return NULL;
1555         }
1556
1557         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1558         if (!f) {
1559                 free(func_name);
1560                 return NULL;
1561         }
1562
1563         if (function)
1564                 *function = func;
1565
1566         free(func_name);
1567         return f;
1568 }
1569
1570 int
1571 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1572                                       const char *name,
1573                                       const char *mailbox_struct_type_name,
1574                                       rte_swx_extern_func_t func)
1575 {
1576         struct extern_func *f;
1577         struct struct_type *mailbox_struct_type;
1578
1579         CHECK(p, EINVAL);
1580
1581         CHECK_NAME(name, EINVAL);
1582         CHECK(!extern_func_find(p, name), EEXIST);
1583
1584         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1585         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1586         CHECK(mailbox_struct_type, EINVAL);
1587
1588         CHECK(func, EINVAL);
1589
1590         /* Node allocation. */
1591         f = calloc(1, sizeof(struct extern_func));
1592         CHECK(func, ENOMEM);
1593
1594         /* Node initialization. */
1595         strcpy(f->name, name);
1596         f->mailbox_struct_type = mailbox_struct_type;
1597         f->func = func;
1598         f->struct_id = p->n_structs;
1599         f->id = p->n_extern_funcs;
1600
1601         /* Node add to tailq. */
1602         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1603         p->n_extern_funcs++;
1604         p->n_structs++;
1605
1606         return 0;
1607 }
1608
1609 static int
1610 extern_func_build(struct rte_swx_pipeline *p)
1611 {
1612         uint32_t i;
1613
1614         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1615                 struct thread *t = &p->threads[i];
1616                 struct extern_func *func;
1617
1618                 /* Memory allocation. */
1619                 t->extern_funcs = calloc(p->n_extern_funcs,
1620                                          sizeof(struct extern_func_runtime));
1621                 CHECK(t->extern_funcs, ENOMEM);
1622
1623                 /* Extern function. */
1624                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1625                         struct extern_func_runtime *r =
1626                                 &t->extern_funcs[func->id];
1627                         uint32_t mailbox_size =
1628                                 func->mailbox_struct_type->n_bits / 8;
1629
1630                         r->func = func->func;
1631
1632                         r->mailbox = calloc(1, mailbox_size);
1633                         CHECK(r->mailbox, ENOMEM);
1634
1635                         t->structs[func->struct_id] = r->mailbox;
1636                 }
1637         }
1638
1639         return 0;
1640 }
1641
1642 static void
1643 extern_func_build_free(struct rte_swx_pipeline *p)
1644 {
1645         uint32_t i;
1646
1647         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1648                 struct thread *t = &p->threads[i];
1649                 uint32_t j;
1650
1651                 if (!t->extern_funcs)
1652                         continue;
1653
1654                 for (j = 0; j < p->n_extern_funcs; j++) {
1655                         struct extern_func_runtime *r = &t->extern_funcs[j];
1656
1657                         free(r->mailbox);
1658                 }
1659
1660                 free(t->extern_funcs);
1661                 t->extern_funcs = NULL;
1662         }
1663 }
1664
1665 static void
1666 extern_func_free(struct rte_swx_pipeline *p)
1667 {
1668         extern_func_build_free(p);
1669
1670         for ( ; ; ) {
1671                 struct extern_func *elem;
1672
1673                 elem = TAILQ_FIRST(&p->extern_funcs);
1674                 if (!elem)
1675                         break;
1676
1677                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1678                 free(elem);
1679         }
1680 }
1681
1682 /*
1683  * Header.
1684  */
1685 static struct header *
1686 header_find(struct rte_swx_pipeline *p, const char *name)
1687 {
1688         struct header *elem;
1689
1690         TAILQ_FOREACH(elem, &p->headers, node)
1691                 if (strcmp(elem->name, name) == 0)
1692                         return elem;
1693
1694         return NULL;
1695 }
1696
1697 static struct header *
1698 header_parse(struct rte_swx_pipeline *p,
1699              const char *name)
1700 {
1701         if (name[0] != 'h' || name[1] != '.')
1702                 return NULL;
1703
1704         return header_find(p, &name[2]);
1705 }
1706
1707 static struct field *
1708 header_field_parse(struct rte_swx_pipeline *p,
1709                    const char *name,
1710                    struct header **header)
1711 {
1712         struct header *h;
1713         struct field *f;
1714         char *header_name, *field_name;
1715
1716         if ((name[0] != 'h') || (name[1] != '.'))
1717                 return NULL;
1718
1719         header_name = strdup(&name[2]);
1720         if (!header_name)
1721                 return NULL;
1722
1723         field_name = strchr(header_name, '.');
1724         if (!field_name) {
1725                 free(header_name);
1726                 return NULL;
1727         }
1728
1729         *field_name = 0;
1730         field_name++;
1731
1732         h = header_find(p, header_name);
1733         if (!h) {
1734                 free(header_name);
1735                 return NULL;
1736         }
1737
1738         f = struct_type_field_find(h->st, field_name);
1739         if (!f) {
1740                 free(header_name);
1741                 return NULL;
1742         }
1743
1744         if (header)
1745                 *header = h;
1746
1747         free(header_name);
1748         return f;
1749 }
1750
1751 int
1752 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1753                                         const char *name,
1754                                         const char *struct_type_name)
1755 {
1756         struct struct_type *st;
1757         struct header *h;
1758         size_t n_headers_max;
1759
1760         CHECK(p, EINVAL);
1761         CHECK_NAME(name, EINVAL);
1762         CHECK_NAME(struct_type_name, EINVAL);
1763
1764         CHECK(!header_find(p, name), EEXIST);
1765
1766         st = struct_type_find(p, struct_type_name);
1767         CHECK(st, EINVAL);
1768
1769         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1770         CHECK(p->n_headers < n_headers_max, ENOSPC);
1771
1772         /* Node allocation. */
1773         h = calloc(1, sizeof(struct header));
1774         CHECK(h, ENOMEM);
1775
1776         /* Node initialization. */
1777         strcpy(h->name, name);
1778         h->st = st;
1779         h->struct_id = p->n_structs;
1780         h->id = p->n_headers;
1781
1782         /* Node add to tailq. */
1783         TAILQ_INSERT_TAIL(&p->headers, h, node);
1784         p->n_headers++;
1785         p->n_structs++;
1786
1787         return 0;
1788 }
1789
1790 static int
1791 header_build(struct rte_swx_pipeline *p)
1792 {
1793         struct header *h;
1794         uint32_t n_bytes = 0, i;
1795
1796         TAILQ_FOREACH(h, &p->headers, node) {
1797                 n_bytes += h->st->n_bits / 8;
1798         }
1799
1800         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1801                 struct thread *t = &p->threads[i];
1802                 uint32_t offset = 0;
1803
1804                 t->headers = calloc(p->n_headers,
1805                                     sizeof(struct header_runtime));
1806                 CHECK(t->headers, ENOMEM);
1807
1808                 t->headers_out = calloc(p->n_headers,
1809                                         sizeof(struct header_out_runtime));
1810                 CHECK(t->headers_out, ENOMEM);
1811
1812                 t->header_storage = calloc(1, n_bytes);
1813                 CHECK(t->header_storage, ENOMEM);
1814
1815                 t->header_out_storage = calloc(1, n_bytes);
1816                 CHECK(t->header_out_storage, ENOMEM);
1817
1818                 TAILQ_FOREACH(h, &p->headers, node) {
1819                         uint8_t *header_storage;
1820
1821                         header_storage = &t->header_storage[offset];
1822                         offset += h->st->n_bits / 8;
1823
1824                         t->headers[h->id].ptr0 = header_storage;
1825                         t->structs[h->struct_id] = header_storage;
1826                 }
1827         }
1828
1829         return 0;
1830 }
1831
1832 static void
1833 header_build_free(struct rte_swx_pipeline *p)
1834 {
1835         uint32_t i;
1836
1837         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1838                 struct thread *t = &p->threads[i];
1839
1840                 free(t->headers_out);
1841                 t->headers_out = NULL;
1842
1843                 free(t->headers);
1844                 t->headers = NULL;
1845
1846                 free(t->header_out_storage);
1847                 t->header_out_storage = NULL;
1848
1849                 free(t->header_storage);
1850                 t->header_storage = NULL;
1851         }
1852 }
1853
1854 static void
1855 header_free(struct rte_swx_pipeline *p)
1856 {
1857         header_build_free(p);
1858
1859         for ( ; ; ) {
1860                 struct header *elem;
1861
1862                 elem = TAILQ_FIRST(&p->headers);
1863                 if (!elem)
1864                         break;
1865
1866                 TAILQ_REMOVE(&p->headers, elem, node);
1867                 free(elem);
1868         }
1869 }
1870
1871 /*
1872  * Meta-data.
1873  */
1874 static struct field *
1875 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1876 {
1877         if (!p->metadata_st)
1878                 return NULL;
1879
1880         if (name[0] != 'm' || name[1] != '.')
1881                 return NULL;
1882
1883         return struct_type_field_find(p->metadata_st, &name[2]);
1884 }
1885
1886 int
1887 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1888                                           const char *struct_type_name)
1889 {
1890         struct struct_type *st = NULL;
1891
1892         CHECK(p, EINVAL);
1893
1894         CHECK_NAME(struct_type_name, EINVAL);
1895         st  = struct_type_find(p, struct_type_name);
1896         CHECK(st, EINVAL);
1897         CHECK(!p->metadata_st, EINVAL);
1898
1899         p->metadata_st = st;
1900         p->metadata_struct_id = p->n_structs;
1901
1902         p->n_structs++;
1903
1904         return 0;
1905 }
1906
1907 static int
1908 metadata_build(struct rte_swx_pipeline *p)
1909 {
1910         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1911         uint32_t i;
1912
1913         /* Thread-level initialization. */
1914         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1915                 struct thread *t = &p->threads[i];
1916                 uint8_t *metadata;
1917
1918                 metadata = calloc(1, n_bytes);
1919                 CHECK(metadata, ENOMEM);
1920
1921                 t->metadata = metadata;
1922                 t->structs[p->metadata_struct_id] = metadata;
1923         }
1924
1925         return 0;
1926 }
1927
1928 static void
1929 metadata_build_free(struct rte_swx_pipeline *p)
1930 {
1931         uint32_t i;
1932
1933         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1934                 struct thread *t = &p->threads[i];
1935
1936                 free(t->metadata);
1937                 t->metadata = NULL;
1938         }
1939 }
1940
1941 static void
1942 metadata_free(struct rte_swx_pipeline *p)
1943 {
1944         metadata_build_free(p);
1945 }
1946
1947 /*
1948  * Instruction.
1949  */
1950 static struct field *
1951 action_field_parse(struct action *action, const char *name);
1952
1953 static struct field *
1954 struct_field_parse(struct rte_swx_pipeline *p,
1955                    struct action *action,
1956                    const char *name,
1957                    uint32_t *struct_id)
1958 {
1959         struct field *f;
1960
1961         switch (name[0]) {
1962         case 'h':
1963         {
1964                 struct header *header;
1965
1966                 f = header_field_parse(p, name, &header);
1967                 if (!f)
1968                         return NULL;
1969
1970                 *struct_id = header->struct_id;
1971                 return f;
1972         }
1973
1974         case 'm':
1975         {
1976                 f = metadata_field_parse(p, name);
1977                 if (!f)
1978                         return NULL;
1979
1980                 *struct_id = p->metadata_struct_id;
1981                 return f;
1982         }
1983
1984         case 't':
1985         {
1986                 if (!action)
1987                         return NULL;
1988
1989                 f = action_field_parse(action, name);
1990                 if (!f)
1991                         return NULL;
1992
1993                 *struct_id = 0;
1994                 return f;
1995         }
1996
1997         case 'e':
1998         {
1999                 struct extern_obj *obj;
2000
2001                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2002                 if (!f)
2003                         return NULL;
2004
2005                 *struct_id = obj->struct_id;
2006                 return f;
2007         }
2008
2009         case 'f':
2010         {
2011                 struct extern_func *func;
2012
2013                 f = extern_func_mailbox_field_parse(p, name, &func);
2014                 if (!f)
2015                         return NULL;
2016
2017                 *struct_id = func->struct_id;
2018                 return f;
2019         }
2020
2021         default:
2022                 return NULL;
2023         }
2024 }
2025
2026 static inline void
2027 pipeline_port_inc(struct rte_swx_pipeline *p)
2028 {
2029         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2030 }
2031
2032 static inline void
2033 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2034 {
2035         t->ip = p->instructions;
2036 }
2037
2038 static inline void
2039 thread_ip_inc(struct rte_swx_pipeline *p);
2040
2041 static inline void
2042 thread_ip_inc(struct rte_swx_pipeline *p)
2043 {
2044         struct thread *t = &p->threads[p->thread_id];
2045
2046         t->ip++;
2047 }
2048
2049 static inline void
2050 thread_ip_inc_cond(struct thread *t, int cond)
2051 {
2052         t->ip += cond;
2053 }
2054
2055 static inline void
2056 thread_yield(struct rte_swx_pipeline *p)
2057 {
2058         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2059 }
2060
2061 /*
2062  * rx.
2063  */
2064 static int
2065 instr_rx_translate(struct rte_swx_pipeline *p,
2066                    struct action *action,
2067                    char **tokens,
2068                    int n_tokens,
2069                    struct instruction *instr,
2070                    struct instruction_data *data __rte_unused)
2071 {
2072         struct field *f;
2073
2074         CHECK(!action, EINVAL);
2075         CHECK(n_tokens == 2, EINVAL);
2076
2077         f = metadata_field_parse(p, tokens[1]);
2078         CHECK(f, EINVAL);
2079
2080         instr->type = INSTR_RX;
2081         instr->io.io.offset = f->offset / 8;
2082         instr->io.io.n_bits = f->n_bits;
2083         return 0;
2084 }
2085
2086 static inline void
2087 instr_rx_exec(struct rte_swx_pipeline *p);
2088
2089 static inline void
2090 instr_rx_exec(struct rte_swx_pipeline *p)
2091 {
2092         struct thread *t = &p->threads[p->thread_id];
2093         struct instruction *ip = t->ip;
2094         struct port_in_runtime *port = &p->in[p->port_id];
2095         struct rte_swx_pkt *pkt = &t->pkt;
2096         int pkt_received;
2097
2098         /* Packet. */
2099         pkt_received = port->pkt_rx(port->obj, pkt);
2100         t->ptr = &pkt->pkt[pkt->offset];
2101         rte_prefetch0(t->ptr);
2102
2103         TRACE("[Thread %2u] rx %s from port %u\n",
2104               p->thread_id,
2105               pkt_received ? "1 pkt" : "0 pkts",
2106               p->port_id);
2107
2108         /* Headers. */
2109         t->valid_headers = 0;
2110         t->n_headers_out = 0;
2111
2112         /* Meta-data. */
2113         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2114
2115         /* Tables. */
2116         t->table_state = p->table_state;
2117
2118         /* Thread. */
2119         pipeline_port_inc(p);
2120         thread_ip_inc_cond(t, pkt_received);
2121         thread_yield(p);
2122 }
2123
2124 /*
2125  * tx.
2126  */
2127 static int
2128 instr_tx_translate(struct rte_swx_pipeline *p,
2129                    struct action *action __rte_unused,
2130                    char **tokens,
2131                    int n_tokens,
2132                    struct instruction *instr,
2133                    struct instruction_data *data __rte_unused)
2134 {
2135         struct field *f;
2136
2137         CHECK(n_tokens == 2, EINVAL);
2138
2139         f = metadata_field_parse(p, tokens[1]);
2140         CHECK(f, EINVAL);
2141
2142         instr->type = INSTR_TX;
2143         instr->io.io.offset = f->offset / 8;
2144         instr->io.io.n_bits = f->n_bits;
2145         return 0;
2146 }
2147
2148 static inline void
2149 emit_handler(struct thread *t)
2150 {
2151         struct header_out_runtime *h0 = &t->headers_out[0];
2152         struct header_out_runtime *h1 = &t->headers_out[1];
2153         uint32_t offset = 0, i;
2154
2155         /* No header change or header decapsulation. */
2156         if ((t->n_headers_out == 1) &&
2157             (h0->ptr + h0->n_bytes == t->ptr)) {
2158                 TRACE("Emit handler: no header change or header decap.\n");
2159
2160                 t->pkt.offset -= h0->n_bytes;
2161                 t->pkt.length += h0->n_bytes;
2162
2163                 return;
2164         }
2165
2166         /* Header encapsulation (optionally, with prior header decasulation). */
2167         if ((t->n_headers_out == 2) &&
2168             (h1->ptr + h1->n_bytes == t->ptr) &&
2169             (h0->ptr == h0->ptr0)) {
2170                 uint32_t offset;
2171
2172                 TRACE("Emit handler: header encapsulation.\n");
2173
2174                 offset = h0->n_bytes + h1->n_bytes;
2175                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2176                 t->pkt.offset -= offset;
2177                 t->pkt.length += offset;
2178
2179                 return;
2180         }
2181
2182         /* Header insertion. */
2183         /* TBD */
2184
2185         /* Header extraction. */
2186         /* TBD */
2187
2188         /* For any other case. */
2189         TRACE("Emit handler: complex case.\n");
2190
2191         for (i = 0; i < t->n_headers_out; i++) {
2192                 struct header_out_runtime *h = &t->headers_out[i];
2193
2194                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2195                 offset += h->n_bytes;
2196         }
2197
2198         if (offset) {
2199                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2200                 t->pkt.offset -= offset;
2201                 t->pkt.length += offset;
2202         }
2203 }
2204
2205 static inline void
2206 instr_tx_exec(struct rte_swx_pipeline *p);
2207
2208 static inline void
2209 instr_tx_exec(struct rte_swx_pipeline *p)
2210 {
2211         struct thread *t = &p->threads[p->thread_id];
2212         struct instruction *ip = t->ip;
2213         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2214         struct port_out_runtime *port = &p->out[port_id];
2215         struct rte_swx_pkt *pkt = &t->pkt;
2216
2217         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2218               p->thread_id,
2219               (uint32_t)port_id);
2220
2221         /* Headers. */
2222         emit_handler(t);
2223
2224         /* Packet. */
2225         port->pkt_tx(port->obj, pkt);
2226
2227         /* Thread. */
2228         thread_ip_reset(p, t);
2229         instr_rx_exec(p);
2230 }
2231
2232 /*
2233  * extract.
2234  */
2235 static int
2236 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2237                             struct action *action,
2238                             char **tokens,
2239                             int n_tokens,
2240                             struct instruction *instr,
2241                             struct instruction_data *data __rte_unused)
2242 {
2243         struct header *h;
2244
2245         CHECK(!action, EINVAL);
2246         CHECK(n_tokens == 2, EINVAL);
2247
2248         h = header_parse(p, tokens[1]);
2249         CHECK(h, EINVAL);
2250
2251         instr->type = INSTR_HDR_EXTRACT;
2252         instr->io.hdr.header_id[0] = h->id;
2253         instr->io.hdr.struct_id[0] = h->struct_id;
2254         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2255         return 0;
2256 }
2257
2258 static inline void
2259 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2260
2261 static inline void
2262 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2263 {
2264         struct thread *t = &p->threads[p->thread_id];
2265         struct instruction *ip = t->ip;
2266         uint64_t valid_headers = t->valid_headers;
2267         uint8_t *ptr = t->ptr;
2268         uint32_t offset = t->pkt.offset;
2269         uint32_t length = t->pkt.length;
2270         uint32_t i;
2271
2272         for (i = 0; i < n_extract; i++) {
2273                 uint32_t header_id = ip->io.hdr.header_id[i];
2274                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2275                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2276
2277                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2278                       p->thread_id,
2279                       header_id,
2280                       n_bytes);
2281
2282                 /* Headers. */
2283                 t->structs[struct_id] = ptr;
2284                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2285
2286                 /* Packet. */
2287                 offset += n_bytes;
2288                 length -= n_bytes;
2289                 ptr += n_bytes;
2290         }
2291
2292         /* Headers. */
2293         t->valid_headers = valid_headers;
2294
2295         /* Packet. */
2296         t->pkt.offset = offset;
2297         t->pkt.length = length;
2298         t->ptr = ptr;
2299 }
2300
2301 static inline void
2302 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2303 {
2304         __instr_hdr_extract_exec(p, 1);
2305
2306         /* Thread. */
2307         thread_ip_inc(p);
2308 }
2309
2310 static inline void
2311 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2312 {
2313         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2314               p->thread_id);
2315
2316         __instr_hdr_extract_exec(p, 2);
2317
2318         /* Thread. */
2319         thread_ip_inc(p);
2320 }
2321
2322 static inline void
2323 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2324 {
2325         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2326               p->thread_id);
2327
2328         __instr_hdr_extract_exec(p, 3);
2329
2330         /* Thread. */
2331         thread_ip_inc(p);
2332 }
2333
2334 static inline void
2335 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2336 {
2337         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2338               p->thread_id);
2339
2340         __instr_hdr_extract_exec(p, 4);
2341
2342         /* Thread. */
2343         thread_ip_inc(p);
2344 }
2345
2346 static inline void
2347 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2348 {
2349         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2350               p->thread_id);
2351
2352         __instr_hdr_extract_exec(p, 5);
2353
2354         /* Thread. */
2355         thread_ip_inc(p);
2356 }
2357
2358 static inline void
2359 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2360 {
2361         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2362               p->thread_id);
2363
2364         __instr_hdr_extract_exec(p, 6);
2365
2366         /* Thread. */
2367         thread_ip_inc(p);
2368 }
2369
2370 static inline void
2371 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2372 {
2373         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2374               p->thread_id);
2375
2376         __instr_hdr_extract_exec(p, 7);
2377
2378         /* Thread. */
2379         thread_ip_inc(p);
2380 }
2381
2382 static inline void
2383 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2384 {
2385         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2386               p->thread_id);
2387
2388         __instr_hdr_extract_exec(p, 8);
2389
2390         /* Thread. */
2391         thread_ip_inc(p);
2392 }
2393
2394 /*
2395  * emit.
2396  */
2397 static int
2398 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2399                          struct action *action __rte_unused,
2400                          char **tokens,
2401                          int n_tokens,
2402                          struct instruction *instr,
2403                          struct instruction_data *data __rte_unused)
2404 {
2405         struct header *h;
2406
2407         CHECK(n_tokens == 2, EINVAL);
2408
2409         h = header_parse(p, tokens[1]);
2410         CHECK(h, EINVAL);
2411
2412         instr->type = INSTR_HDR_EMIT;
2413         instr->io.hdr.header_id[0] = h->id;
2414         instr->io.hdr.struct_id[0] = h->struct_id;
2415         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2416         return 0;
2417 }
2418
2419 static inline void
2420 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
2421
2422 static inline void
2423 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
2424 {
2425         struct thread *t = &p->threads[p->thread_id];
2426         struct instruction *ip = t->ip;
2427         uint32_t n_headers_out = t->n_headers_out;
2428         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
2429         uint8_t *ho_ptr = NULL;
2430         uint32_t ho_nbytes = 0, i;
2431
2432         for (i = 0; i < n_emit; i++) {
2433                 uint32_t header_id = ip->io.hdr.header_id[i];
2434                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2435                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2436
2437                 struct header_runtime *hi = &t->headers[header_id];
2438                 uint8_t *hi_ptr = t->structs[struct_id];
2439
2440                 TRACE("[Thread %2u]: emit header %u\n",
2441                       p->thread_id,
2442                       header_id);
2443
2444                 /* Headers. */
2445                 if (!i) {
2446                         if (!t->n_headers_out) {
2447                                 ho = &t->headers_out[0];
2448
2449                                 ho->ptr0 = hi->ptr0;
2450                                 ho->ptr = hi_ptr;
2451
2452                                 ho_ptr = hi_ptr;
2453                                 ho_nbytes = n_bytes;
2454
2455                                 n_headers_out = 1;
2456
2457                                 continue;
2458                         } else {
2459                                 ho_ptr = ho->ptr;
2460                                 ho_nbytes = ho->n_bytes;
2461                         }
2462                 }
2463
2464                 if (ho_ptr + ho_nbytes == hi_ptr) {
2465                         ho_nbytes += n_bytes;
2466                 } else {
2467                         ho->n_bytes = ho_nbytes;
2468
2469                         ho++;
2470                         ho->ptr0 = hi->ptr0;
2471                         ho->ptr = hi_ptr;
2472
2473                         ho_ptr = hi_ptr;
2474                         ho_nbytes = n_bytes;
2475
2476                         n_headers_out++;
2477                 }
2478         }
2479
2480         ho->n_bytes = ho_nbytes;
2481         t->n_headers_out = n_headers_out;
2482 }
2483
2484 static inline void
2485 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2486 {
2487         __instr_hdr_emit_exec(p, 1);
2488
2489         /* Thread. */
2490         thread_ip_inc(p);
2491 }
2492
2493 static inline void
2494 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2495 {
2496         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2497               p->thread_id);
2498
2499         __instr_hdr_emit_exec(p, 1);
2500         instr_tx_exec(p);
2501 }
2502
2503 static inline void
2504 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2505 {
2506         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2507               p->thread_id);
2508
2509         __instr_hdr_emit_exec(p, 2);
2510         instr_tx_exec(p);
2511 }
2512
2513 static inline void
2514 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2515 {
2516         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2517               p->thread_id);
2518
2519         __instr_hdr_emit_exec(p, 3);
2520         instr_tx_exec(p);
2521 }
2522
2523 static inline void
2524 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2525 {
2526         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2527               p->thread_id);
2528
2529         __instr_hdr_emit_exec(p, 4);
2530         instr_tx_exec(p);
2531 }
2532
2533 static inline void
2534 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2535 {
2536         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2537               p->thread_id);
2538
2539         __instr_hdr_emit_exec(p, 5);
2540         instr_tx_exec(p);
2541 }
2542
2543 static inline void
2544 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2545 {
2546         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2547               p->thread_id);
2548
2549         __instr_hdr_emit_exec(p, 6);
2550         instr_tx_exec(p);
2551 }
2552
2553 static inline void
2554 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2555 {
2556         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2557               p->thread_id);
2558
2559         __instr_hdr_emit_exec(p, 7);
2560         instr_tx_exec(p);
2561 }
2562
2563 static inline void
2564 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2565 {
2566         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2567               p->thread_id);
2568
2569         __instr_hdr_emit_exec(p, 8);
2570         instr_tx_exec(p);
2571 }
2572
2573 /*
2574  * validate.
2575  */
2576 static int
2577 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2578                              struct action *action __rte_unused,
2579                              char **tokens,
2580                              int n_tokens,
2581                              struct instruction *instr,
2582                              struct instruction_data *data __rte_unused)
2583 {
2584         struct header *h;
2585
2586         CHECK(n_tokens == 2, EINVAL);
2587
2588         h = header_parse(p, tokens[1]);
2589         CHECK(h, EINVAL);
2590
2591         instr->type = INSTR_HDR_VALIDATE;
2592         instr->valid.header_id = h->id;
2593         return 0;
2594 }
2595
2596 static inline void
2597 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2598 {
2599         struct thread *t = &p->threads[p->thread_id];
2600         struct instruction *ip = t->ip;
2601         uint32_t header_id = ip->valid.header_id;
2602
2603         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2604
2605         /* Headers. */
2606         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2607
2608         /* Thread. */
2609         thread_ip_inc(p);
2610 }
2611
2612 /*
2613  * invalidate.
2614  */
2615 static int
2616 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2617                                struct action *action __rte_unused,
2618                                char **tokens,
2619                                int n_tokens,
2620                                struct instruction *instr,
2621                                struct instruction_data *data __rte_unused)
2622 {
2623         struct header *h;
2624
2625         CHECK(n_tokens == 2, EINVAL);
2626
2627         h = header_parse(p, tokens[1]);
2628         CHECK(h, EINVAL);
2629
2630         instr->type = INSTR_HDR_INVALIDATE;
2631         instr->valid.header_id = h->id;
2632         return 0;
2633 }
2634
2635 static inline void
2636 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2637 {
2638         struct thread *t = &p->threads[p->thread_id];
2639         struct instruction *ip = t->ip;
2640         uint32_t header_id = ip->valid.header_id;
2641
2642         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2643
2644         /* Headers. */
2645         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2646
2647         /* Thread. */
2648         thread_ip_inc(p);
2649 }
2650
2651 /*
2652  * mov.
2653  */
2654 static int
2655 instr_mov_translate(struct rte_swx_pipeline *p,
2656                     struct action *action,
2657                     char **tokens,
2658                     int n_tokens,
2659                     struct instruction *instr,
2660                     struct instruction_data *data __rte_unused)
2661 {
2662         char *dst = tokens[1], *src = tokens[2];
2663         struct field *fdst, *fsrc;
2664         uint32_t dst_struct_id, src_struct_id, src_val;
2665
2666         CHECK(n_tokens == 3, EINVAL);
2667
2668         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2669         CHECK(fdst, EINVAL);
2670
2671         /* MOV or MOV_S. */
2672         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2673         if (fsrc) {
2674                 instr->type = INSTR_MOV;
2675                 if ((dst[0] == 'h' && src[0] != 'h') ||
2676                     (dst[0] != 'h' && src[0] == 'h'))
2677                         instr->type = INSTR_MOV_S;
2678
2679                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2680                 instr->mov.dst.n_bits = fdst->n_bits;
2681                 instr->mov.dst.offset = fdst->offset / 8;
2682                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2683                 instr->mov.src.n_bits = fsrc->n_bits;
2684                 instr->mov.src.offset = fsrc->offset / 8;
2685                 return 0;
2686         }
2687
2688         /* MOV_I. */
2689         src_val = strtoul(src, &src, 0);
2690         CHECK(!src[0], EINVAL);
2691
2692         if (dst[0] == 'h')
2693                 src_val = htonl(src_val);
2694
2695         instr->type = INSTR_MOV_I;
2696         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2697         instr->mov.dst.n_bits = fdst->n_bits;
2698         instr->mov.dst.offset = fdst->offset / 8;
2699         instr->mov.src_val = (uint32_t)src_val;
2700         return 0;
2701 }
2702
2703 static inline void
2704 instr_mov_exec(struct rte_swx_pipeline *p)
2705 {
2706         struct thread *t = &p->threads[p->thread_id];
2707         struct instruction *ip = t->ip;
2708
2709         TRACE("[Thread %2u] mov\n",
2710               p->thread_id);
2711
2712         MOV(t, ip);
2713
2714         /* Thread. */
2715         thread_ip_inc(p);
2716 }
2717
2718 static inline void
2719 instr_mov_s_exec(struct rte_swx_pipeline *p)
2720 {
2721         struct thread *t = &p->threads[p->thread_id];
2722         struct instruction *ip = t->ip;
2723
2724         TRACE("[Thread %2u] mov (s)\n",
2725               p->thread_id);
2726
2727         MOV_S(t, ip);
2728
2729         /* Thread. */
2730         thread_ip_inc(p);
2731 }
2732
2733 static inline void
2734 instr_mov_i_exec(struct rte_swx_pipeline *p)
2735 {
2736         struct thread *t = &p->threads[p->thread_id];
2737         struct instruction *ip = t->ip;
2738
2739         TRACE("[Thread %2u] mov m.f %x\n",
2740               p->thread_id,
2741               ip->mov.src_val);
2742
2743         MOV_I(t, ip);
2744
2745         /* Thread. */
2746         thread_ip_inc(p);
2747 }
2748
2749 /*
2750  * dma.
2751  */
2752 static int
2753 instr_dma_translate(struct rte_swx_pipeline *p,
2754                     struct action *action,
2755                     char **tokens,
2756                     int n_tokens,
2757                     struct instruction *instr,
2758                     struct instruction_data *data __rte_unused)
2759 {
2760         char *dst = tokens[1];
2761         char *src = tokens[2];
2762         struct header *h;
2763         struct field *tf;
2764
2765         CHECK(action, EINVAL);
2766         CHECK(n_tokens == 3, EINVAL);
2767
2768         h = header_parse(p, dst);
2769         CHECK(h, EINVAL);
2770
2771         tf = action_field_parse(action, src);
2772         CHECK(tf, EINVAL);
2773
2774         instr->type = INSTR_DMA_HT;
2775         instr->dma.dst.header_id[0] = h->id;
2776         instr->dma.dst.struct_id[0] = h->struct_id;
2777         instr->dma.n_bytes[0] = h->st->n_bits / 8;
2778         instr->dma.src.offset[0] = tf->offset / 8;
2779
2780         return 0;
2781 }
2782
2783 static inline void
2784 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2785
2786 static inline void
2787 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2788 {
2789         struct thread *t = &p->threads[p->thread_id];
2790         struct instruction *ip = t->ip;
2791         uint8_t *action_data = t->structs[0];
2792         uint64_t valid_headers = t->valid_headers;
2793         uint32_t i;
2794
2795         for (i = 0; i < n_dma; i++) {
2796                 uint32_t header_id = ip->dma.dst.header_id[i];
2797                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2798                 uint32_t offset = ip->dma.src.offset[i];
2799                 uint32_t n_bytes = ip->dma.n_bytes[i];
2800
2801                 struct header_runtime *h = &t->headers[header_id];
2802                 uint8_t *h_ptr0 = h->ptr0;
2803                 uint8_t *h_ptr = t->structs[struct_id];
2804
2805                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2806                         h_ptr : h_ptr0;
2807                 void *src = &action_data[offset];
2808
2809                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2810
2811                 /* Headers. */
2812                 memcpy(dst, src, n_bytes);
2813                 t->structs[struct_id] = dst;
2814                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2815         }
2816
2817         t->valid_headers = valid_headers;
2818 }
2819
2820 static inline void
2821 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2822 {
2823         __instr_dma_ht_exec(p, 1);
2824
2825         /* Thread. */
2826         thread_ip_inc(p);
2827 }
2828
2829 static inline void
2830 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2831 {
2832         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2833               p->thread_id);
2834
2835         __instr_dma_ht_exec(p, 2);
2836
2837         /* Thread. */
2838         thread_ip_inc(p);
2839 }
2840
2841 static inline void
2842 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2843 {
2844         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2845               p->thread_id);
2846
2847         __instr_dma_ht_exec(p, 3);
2848
2849         /* Thread. */
2850         thread_ip_inc(p);
2851 }
2852
2853 static inline void
2854 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2855 {
2856         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2857               p->thread_id);
2858
2859         __instr_dma_ht_exec(p, 4);
2860
2861         /* Thread. */
2862         thread_ip_inc(p);
2863 }
2864
2865 static inline void
2866 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2867 {
2868         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2869               p->thread_id);
2870
2871         __instr_dma_ht_exec(p, 5);
2872
2873         /* Thread. */
2874         thread_ip_inc(p);
2875 }
2876
2877 static inline void
2878 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2879 {
2880         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2881               p->thread_id);
2882
2883         __instr_dma_ht_exec(p, 6);
2884
2885         /* Thread. */
2886         thread_ip_inc(p);
2887 }
2888
2889 static inline void
2890 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2891 {
2892         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2893               p->thread_id);
2894
2895         __instr_dma_ht_exec(p, 7);
2896
2897         /* Thread. */
2898         thread_ip_inc(p);
2899 }
2900
2901 static inline void
2902 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2903 {
2904         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2905               p->thread_id);
2906
2907         __instr_dma_ht_exec(p, 8);
2908
2909         /* Thread. */
2910         thread_ip_inc(p);
2911 }
2912
2913 /*
2914  * alu.
2915  */
2916 static int
2917 instr_alu_add_translate(struct rte_swx_pipeline *p,
2918                         struct action *action,
2919                         char **tokens,
2920                         int n_tokens,
2921                         struct instruction *instr,
2922                         struct instruction_data *data __rte_unused)
2923 {
2924         char *dst = tokens[1], *src = tokens[2];
2925         struct field *fdst, *fsrc;
2926         uint32_t dst_struct_id, src_struct_id, src_val;
2927
2928         CHECK(n_tokens == 3, EINVAL);
2929
2930         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2931         CHECK(fdst, EINVAL);
2932
2933         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2934         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2935         if (fsrc) {
2936                 instr->type = INSTR_ALU_ADD;
2937                 if (dst[0] == 'h' && src[0] == 'm')
2938                         instr->type = INSTR_ALU_ADD_HM;
2939                 if (dst[0] == 'm' && src[0] == 'h')
2940                         instr->type = INSTR_ALU_ADD_MH;
2941                 if (dst[0] == 'h' && src[0] == 'h')
2942                         instr->type = INSTR_ALU_ADD_HH;
2943
2944                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2945                 instr->alu.dst.n_bits = fdst->n_bits;
2946                 instr->alu.dst.offset = fdst->offset / 8;
2947                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2948                 instr->alu.src.n_bits = fsrc->n_bits;
2949                 instr->alu.src.offset = fsrc->offset / 8;
2950                 return 0;
2951         }
2952
2953         /* ADD_MI, ADD_HI. */
2954         src_val = strtoul(src, &src, 0);
2955         CHECK(!src[0], EINVAL);
2956
2957         instr->type = INSTR_ALU_ADD_MI;
2958         if (dst[0] == 'h')
2959                 instr->type = INSTR_ALU_ADD_HI;
2960
2961         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2962         instr->alu.dst.n_bits = fdst->n_bits;
2963         instr->alu.dst.offset = fdst->offset / 8;
2964         instr->alu.src_val = (uint32_t)src_val;
2965         return 0;
2966 }
2967
2968 static int
2969 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2970                         struct action *action,
2971                         char **tokens,
2972                         int n_tokens,
2973                         struct instruction *instr,
2974                         struct instruction_data *data __rte_unused)
2975 {
2976         char *dst = tokens[1], *src = tokens[2];
2977         struct field *fdst, *fsrc;
2978         uint32_t dst_struct_id, src_struct_id, src_val;
2979
2980         CHECK(n_tokens == 3, EINVAL);
2981
2982         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2983         CHECK(fdst, EINVAL);
2984
2985         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2986         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2987         if (fsrc) {
2988                 instr->type = INSTR_ALU_SUB;
2989                 if (dst[0] == 'h' && src[0] == 'm')
2990                         instr->type = INSTR_ALU_SUB_HM;
2991                 if (dst[0] == 'm' && src[0] == 'h')
2992                         instr->type = INSTR_ALU_SUB_MH;
2993                 if (dst[0] == 'h' && src[0] == 'h')
2994                         instr->type = INSTR_ALU_SUB_HH;
2995
2996                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2997                 instr->alu.dst.n_bits = fdst->n_bits;
2998                 instr->alu.dst.offset = fdst->offset / 8;
2999                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3000                 instr->alu.src.n_bits = fsrc->n_bits;
3001                 instr->alu.src.offset = fsrc->offset / 8;
3002                 return 0;
3003         }
3004
3005         /* SUB_MI, SUB_HI. */
3006         src_val = strtoul(src, &src, 0);
3007         CHECK(!src[0], EINVAL);
3008
3009         instr->type = INSTR_ALU_SUB_MI;
3010         if (dst[0] == 'h')
3011                 instr->type = INSTR_ALU_SUB_HI;
3012
3013         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3014         instr->alu.dst.n_bits = fdst->n_bits;
3015         instr->alu.dst.offset = fdst->offset / 8;
3016         instr->alu.src_val = (uint32_t)src_val;
3017         return 0;
3018 }
3019
3020 static int
3021 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3022                           struct action *action __rte_unused,
3023                           char **tokens,
3024                           int n_tokens,
3025                           struct instruction *instr,
3026                           struct instruction_data *data __rte_unused)
3027 {
3028         char *dst = tokens[1], *src = tokens[2];
3029         struct header *hdst, *hsrc;
3030         struct field *fdst, *fsrc;
3031
3032         CHECK(n_tokens == 3, EINVAL);
3033
3034         fdst = header_field_parse(p, dst, &hdst);
3035         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3036
3037         /* CKADD_FIELD. */
3038         fsrc = header_field_parse(p, src, &hsrc);
3039         if (fsrc) {
3040                 instr->type = INSTR_ALU_CKADD_FIELD;
3041                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3042                 instr->alu.dst.n_bits = fdst->n_bits;
3043                 instr->alu.dst.offset = fdst->offset / 8;
3044                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3045                 instr->alu.src.n_bits = fsrc->n_bits;
3046                 instr->alu.src.offset = fsrc->offset / 8;
3047                 return 0;
3048         }
3049
3050         /* CKADD_STRUCT, CKADD_STRUCT20. */
3051         hsrc = header_parse(p, src);
3052         CHECK(hsrc, EINVAL);
3053
3054         instr->type = INSTR_ALU_CKADD_STRUCT;
3055         if ((hsrc->st->n_bits / 8) == 20)
3056                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3057
3058         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3059         instr->alu.dst.n_bits = fdst->n_bits;
3060         instr->alu.dst.offset = fdst->offset / 8;
3061         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3062         instr->alu.src.n_bits = hsrc->st->n_bits;
3063         instr->alu.src.offset = 0; /* Unused. */
3064         return 0;
3065 }
3066
3067 static int
3068 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3069                           struct action *action __rte_unused,
3070                           char **tokens,
3071                           int n_tokens,
3072                           struct instruction *instr,
3073                           struct instruction_data *data __rte_unused)
3074 {
3075         char *dst = tokens[1], *src = tokens[2];
3076         struct header *hdst, *hsrc;
3077         struct field *fdst, *fsrc;
3078
3079         CHECK(n_tokens == 3, EINVAL);
3080
3081         fdst = header_field_parse(p, dst, &hdst);
3082         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3083
3084         fsrc = header_field_parse(p, src, &hsrc);
3085         CHECK(fsrc, EINVAL);
3086
3087         instr->type = INSTR_ALU_CKSUB_FIELD;
3088         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3089         instr->alu.dst.n_bits = fdst->n_bits;
3090         instr->alu.dst.offset = fdst->offset / 8;
3091         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3092         instr->alu.src.n_bits = fsrc->n_bits;
3093         instr->alu.src.offset = fsrc->offset / 8;
3094         return 0;
3095 }
3096
3097 static int
3098 instr_alu_and_translate(struct rte_swx_pipeline *p,
3099                         struct action *action,
3100                         char **tokens,
3101                         int n_tokens,
3102                         struct instruction *instr,
3103                         struct instruction_data *data __rte_unused)
3104 {
3105         char *dst = tokens[1], *src = tokens[2];
3106         struct field *fdst, *fsrc;
3107         uint32_t dst_struct_id, src_struct_id, src_val;
3108
3109         CHECK(n_tokens == 3, EINVAL);
3110
3111         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3112         CHECK(fdst, EINVAL);
3113
3114         /* AND or AND_S. */
3115         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3116         if (fsrc) {
3117                 instr->type = INSTR_ALU_AND;
3118                 if ((dst[0] == 'h' && src[0] != 'h') ||
3119                     (dst[0] != 'h' && src[0] == 'h'))
3120                         instr->type = INSTR_ALU_AND_S;
3121
3122                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3123                 instr->alu.dst.n_bits = fdst->n_bits;
3124                 instr->alu.dst.offset = fdst->offset / 8;
3125                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3126                 instr->alu.src.n_bits = fsrc->n_bits;
3127                 instr->alu.src.offset = fsrc->offset / 8;
3128                 return 0;
3129         }
3130
3131         /* AND_I. */
3132         src_val = strtoul(src, &src, 0);
3133         CHECK(!src[0], EINVAL);
3134
3135         if (dst[0] == 'h')
3136                 src_val = htonl(src_val);
3137
3138         instr->type = INSTR_ALU_AND_I;
3139         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3140         instr->alu.dst.n_bits = fdst->n_bits;
3141         instr->alu.dst.offset = fdst->offset / 8;
3142         instr->alu.src_val = (uint32_t)src_val;
3143         return 0;
3144 }
3145
3146 static int
3147 instr_alu_or_translate(struct rte_swx_pipeline *p,
3148                        struct action *action,
3149                        char **tokens,
3150                        int n_tokens,
3151                        struct instruction *instr,
3152                        struct instruction_data *data __rte_unused)
3153 {
3154         char *dst = tokens[1], *src = tokens[2];
3155         struct field *fdst, *fsrc;
3156         uint32_t dst_struct_id, src_struct_id, src_val;
3157
3158         CHECK(n_tokens == 3, EINVAL);
3159
3160         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3161         CHECK(fdst, EINVAL);
3162
3163         /* OR or OR_S. */
3164         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3165         if (fsrc) {
3166                 instr->type = INSTR_ALU_OR;
3167                 if ((dst[0] == 'h' && src[0] != 'h') ||
3168                     (dst[0] != 'h' && src[0] == 'h'))
3169                         instr->type = INSTR_ALU_OR_S;
3170
3171                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3172                 instr->alu.dst.n_bits = fdst->n_bits;
3173                 instr->alu.dst.offset = fdst->offset / 8;
3174                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3175                 instr->alu.src.n_bits = fsrc->n_bits;
3176                 instr->alu.src.offset = fsrc->offset / 8;
3177                 return 0;
3178         }
3179
3180         /* OR_I. */
3181         src_val = strtoul(src, &src, 0);
3182         CHECK(!src[0], EINVAL);
3183
3184         if (dst[0] == 'h')
3185                 src_val = htonl(src_val);
3186
3187         instr->type = INSTR_ALU_OR_I;
3188         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3189         instr->alu.dst.n_bits = fdst->n_bits;
3190         instr->alu.dst.offset = fdst->offset / 8;
3191         instr->alu.src_val = (uint32_t)src_val;
3192         return 0;
3193 }
3194
3195 static int
3196 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3197                         struct action *action,
3198                         char **tokens,
3199                         int n_tokens,
3200                         struct instruction *instr,
3201                         struct instruction_data *data __rte_unused)
3202 {
3203         char *dst = tokens[1], *src = tokens[2];
3204         struct field *fdst, *fsrc;
3205         uint32_t dst_struct_id, src_struct_id, src_val;
3206
3207         CHECK(n_tokens == 3, EINVAL);
3208
3209         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3210         CHECK(fdst, EINVAL);
3211
3212         /* XOR or XOR_S. */
3213         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3214         if (fsrc) {
3215                 instr->type = INSTR_ALU_XOR;
3216                 if ((dst[0] == 'h' && src[0] != 'h') ||
3217                     (dst[0] != 'h' && src[0] == 'h'))
3218                         instr->type = INSTR_ALU_XOR_S;
3219
3220                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3221                 instr->alu.dst.n_bits = fdst->n_bits;
3222                 instr->alu.dst.offset = fdst->offset / 8;
3223                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3224                 instr->alu.src.n_bits = fsrc->n_bits;
3225                 instr->alu.src.offset = fsrc->offset / 8;
3226                 return 0;
3227         }
3228
3229         /* XOR_I. */
3230         src_val = strtoul(src, &src, 0);
3231         CHECK(!src[0], EINVAL);
3232
3233         if (dst[0] == 'h')
3234                 src_val = htonl(src_val);
3235
3236         instr->type = INSTR_ALU_XOR_I;
3237         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3238         instr->alu.dst.n_bits = fdst->n_bits;
3239         instr->alu.dst.offset = fdst->offset / 8;
3240         instr->alu.src_val = (uint32_t)src_val;
3241         return 0;
3242 }
3243
3244 static inline void
3245 instr_alu_add_exec(struct rte_swx_pipeline *p)
3246 {
3247         struct thread *t = &p->threads[p->thread_id];
3248         struct instruction *ip = t->ip;
3249
3250         TRACE("[Thread %2u] add\n", p->thread_id);
3251
3252         /* Structs. */
3253         ALU(t, ip, +);
3254
3255         /* Thread. */
3256         thread_ip_inc(p);
3257 }
3258
3259 static inline void
3260 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3261 {
3262         struct thread *t = &p->threads[p->thread_id];
3263         struct instruction *ip = t->ip;
3264
3265         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3266
3267         /* Structs. */
3268         ALU_MH(t, ip, +);
3269
3270         /* Thread. */
3271         thread_ip_inc(p);
3272 }
3273
3274 static inline void
3275 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3276 {
3277         struct thread *t = &p->threads[p->thread_id];
3278         struct instruction *ip = t->ip;
3279
3280         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3281
3282         /* Structs. */
3283         ALU_HM(t, ip, +);
3284
3285         /* Thread. */
3286         thread_ip_inc(p);
3287 }
3288
3289 static inline void
3290 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3291 {
3292         struct thread *t = &p->threads[p->thread_id];
3293         struct instruction *ip = t->ip;
3294
3295         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3296
3297         /* Structs. */
3298         ALU_HH(t, ip, +);
3299
3300         /* Thread. */
3301         thread_ip_inc(p);
3302 }
3303
3304 static inline void
3305 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3306 {
3307         struct thread *t = &p->threads[p->thread_id];
3308         struct instruction *ip = t->ip;
3309
3310         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3311
3312         /* Structs. */
3313         ALU_MI(t, ip, +);
3314
3315         /* Thread. */
3316         thread_ip_inc(p);
3317 }
3318
3319 static inline void
3320 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3321 {
3322         struct thread *t = &p->threads[p->thread_id];
3323         struct instruction *ip = t->ip;
3324
3325         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3326
3327         /* Structs. */
3328         ALU_HI(t, ip, +);
3329
3330         /* Thread. */
3331         thread_ip_inc(p);
3332 }
3333
3334 static inline void
3335 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3336 {
3337         struct thread *t = &p->threads[p->thread_id];
3338         struct instruction *ip = t->ip;
3339
3340         TRACE("[Thread %2u] sub\n", p->thread_id);
3341
3342         /* Structs. */
3343         ALU(t, ip, -);
3344
3345         /* Thread. */
3346         thread_ip_inc(p);
3347 }
3348
3349 static inline void
3350 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3351 {
3352         struct thread *t = &p->threads[p->thread_id];
3353         struct instruction *ip = t->ip;
3354
3355         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3356
3357         /* Structs. */
3358         ALU_MH(t, ip, -);
3359
3360         /* Thread. */
3361         thread_ip_inc(p);
3362 }
3363
3364 static inline void
3365 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3366 {
3367         struct thread *t = &p->threads[p->thread_id];
3368         struct instruction *ip = t->ip;
3369
3370         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3371
3372         /* Structs. */
3373         ALU_HM(t, ip, -);
3374
3375         /* Thread. */
3376         thread_ip_inc(p);
3377 }
3378
3379 static inline void
3380 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3381 {
3382         struct thread *t = &p->threads[p->thread_id];
3383         struct instruction *ip = t->ip;
3384
3385         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3386
3387         /* Structs. */
3388         ALU_HH(t, ip, -);
3389
3390         /* Thread. */
3391         thread_ip_inc(p);
3392 }
3393
3394 static inline void
3395 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3396 {
3397         struct thread *t = &p->threads[p->thread_id];
3398         struct instruction *ip = t->ip;
3399
3400         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3401
3402         /* Structs. */
3403         ALU_MI(t, ip, -);
3404
3405         /* Thread. */
3406         thread_ip_inc(p);
3407 }
3408
3409 static inline void
3410 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3411 {
3412         struct thread *t = &p->threads[p->thread_id];
3413         struct instruction *ip = t->ip;
3414
3415         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3416
3417         /* Structs. */
3418         ALU_HI(t, ip, -);
3419
3420         /* Thread. */
3421         thread_ip_inc(p);
3422 }
3423
3424 static inline void
3425 instr_alu_and_exec(struct rte_swx_pipeline *p)
3426 {
3427         struct thread *t = &p->threads[p->thread_id];
3428         struct instruction *ip = t->ip;
3429
3430         TRACE("[Thread %2u] and\n", p->thread_id);
3431
3432         /* Structs. */
3433         ALU(t, ip, &);
3434
3435         /* Thread. */
3436         thread_ip_inc(p);
3437 }
3438
3439 static inline void
3440 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
3441 {
3442         struct thread *t = &p->threads[p->thread_id];
3443         struct instruction *ip = t->ip;
3444
3445         TRACE("[Thread %2u] and (s)\n", p->thread_id);
3446
3447         /* Structs. */
3448         ALU_S(t, ip, &);
3449
3450         /* Thread. */
3451         thread_ip_inc(p);
3452 }
3453
3454 static inline void
3455 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3456 {
3457         struct thread *t = &p->threads[p->thread_id];
3458         struct instruction *ip = t->ip;
3459
3460         TRACE("[Thread %2u] and (i)\n", p->thread_id);
3461
3462         /* Structs. */
3463         ALU_I(t, ip, &);
3464
3465         /* Thread. */
3466         thread_ip_inc(p);
3467 }
3468
3469 static inline void
3470 instr_alu_or_exec(struct rte_swx_pipeline *p)
3471 {
3472         struct thread *t = &p->threads[p->thread_id];
3473         struct instruction *ip = t->ip;
3474
3475         TRACE("[Thread %2u] or\n", p->thread_id);
3476
3477         /* Structs. */
3478         ALU(t, ip, |);
3479
3480         /* Thread. */
3481         thread_ip_inc(p);
3482 }
3483
3484 static inline void
3485 instr_alu_or_s_exec(struct rte_swx_pipeline *p)
3486 {
3487         struct thread *t = &p->threads[p->thread_id];
3488         struct instruction *ip = t->ip;
3489
3490         TRACE("[Thread %2u] or (s)\n", p->thread_id);
3491
3492         /* Structs. */
3493         ALU_S(t, ip, |);
3494
3495         /* Thread. */
3496         thread_ip_inc(p);
3497 }
3498
3499 static inline void
3500 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3501 {
3502         struct thread *t = &p->threads[p->thread_id];
3503         struct instruction *ip = t->ip;
3504
3505         TRACE("[Thread %2u] or (i)\n", p->thread_id);
3506
3507         /* Structs. */
3508         ALU_I(t, ip, |);
3509
3510         /* Thread. */
3511         thread_ip_inc(p);
3512 }
3513
3514 static inline void
3515 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3516 {
3517         struct thread *t = &p->threads[p->thread_id];
3518         struct instruction *ip = t->ip;
3519
3520         TRACE("[Thread %2u] xor\n", p->thread_id);
3521
3522         /* Structs. */
3523         ALU(t, ip, ^);
3524
3525         /* Thread. */
3526         thread_ip_inc(p);
3527 }
3528
3529 static inline void
3530 instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
3531 {
3532         struct thread *t = &p->threads[p->thread_id];
3533         struct instruction *ip = t->ip;
3534
3535         TRACE("[Thread %2u] xor (s)\n", p->thread_id);
3536
3537         /* Structs. */
3538         ALU_S(t, ip, ^);
3539
3540         /* Thread. */
3541         thread_ip_inc(p);
3542 }
3543
3544 static inline void
3545 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3546 {
3547         struct thread *t = &p->threads[p->thread_id];
3548         struct instruction *ip = t->ip;
3549
3550         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3551
3552         /* Structs. */
3553         ALU_I(t, ip, ^);
3554
3555         /* Thread. */
3556         thread_ip_inc(p);
3557 }
3558
3559 static inline void
3560 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3561 {
3562         struct thread *t = &p->threads[p->thread_id];
3563         struct instruction *ip = t->ip;
3564         uint8_t *dst_struct, *src_struct;
3565         uint16_t *dst16_ptr, dst;
3566         uint64_t *src64_ptr, src64, src64_mask, src;
3567         uint64_t r;
3568
3569         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3570
3571         /* Structs. */
3572         dst_struct = t->structs[ip->alu.dst.struct_id];
3573         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3574         dst = *dst16_ptr;
3575
3576         src_struct = t->structs[ip->alu.src.struct_id];
3577         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3578         src64 = *src64_ptr;
3579         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3580         src = src64 & src64_mask;
3581
3582         r = dst;
3583         r = ~r & 0xFFFF;
3584
3585         /* The first input (r) is a 16-bit number. The second and the third
3586          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3587          * three numbers (output r) is a 34-bit number.
3588          */
3589         r += (src >> 32) + (src & 0xFFFFFFFF);
3590
3591         /* The first input is a 16-bit number. The second input is an 18-bit
3592          * number. In the worst case scenario, the sum of the two numbers is a
3593          * 19-bit number.
3594          */
3595         r = (r & 0xFFFF) + (r >> 16);
3596
3597         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3598          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3599          */
3600         r = (r & 0xFFFF) + (r >> 16);
3601
3602         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3603          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3604          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3605          * therefore the output r is always a 16-bit number.
3606          */
3607         r = (r & 0xFFFF) + (r >> 16);
3608
3609         r = ~r & 0xFFFF;
3610         r = r ? r : 0xFFFF;
3611
3612         *dst16_ptr = (uint16_t)r;
3613
3614         /* Thread. */
3615         thread_ip_inc(p);
3616 }
3617
3618 static inline void
3619 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3620 {
3621         struct thread *t = &p->threads[p->thread_id];
3622         struct instruction *ip = t->ip;
3623         uint8_t *dst_struct, *src_struct;
3624         uint16_t *dst16_ptr, dst;
3625         uint64_t *src64_ptr, src64, src64_mask, src;
3626         uint64_t r;
3627
3628         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3629
3630         /* Structs. */
3631         dst_struct = t->structs[ip->alu.dst.struct_id];
3632         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3633         dst = *dst16_ptr;
3634
3635         src_struct = t->structs[ip->alu.src.struct_id];
3636         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3637         src64 = *src64_ptr;
3638         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3639         src = src64 & src64_mask;
3640
3641         r = dst;
3642         r = ~r & 0xFFFF;
3643
3644         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3645          * the following sequence of operations in 2's complement arithmetic:
3646          *    a '- b = (a - b) % 0xFFFF.
3647          *
3648          * In order to prevent an underflow for the below subtraction, in which
3649          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3650          * minuend), we first add a multiple of the 0xFFFF modulus to the
3651          * minuend. The number we add to the minuend needs to be a 34-bit number
3652          * or higher, so for readability reasons we picked the 36-bit multiple.
3653          * We are effectively turning the 16-bit minuend into a 36-bit number:
3654          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3655          */
3656         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3657
3658         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3659          * result (the output r) is a 36-bit number.
3660          */
3661         r -= (src >> 32) + (src & 0xFFFFFFFF);
3662
3663         /* The first input is a 16-bit number. The second input is a 20-bit
3664          * number. Their sum is a 21-bit number.
3665          */
3666         r = (r & 0xFFFF) + (r >> 16);
3667
3668         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3669          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3670          */
3671         r = (r & 0xFFFF) + (r >> 16);
3672
3673         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3674          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3675          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3676          * generated, therefore the output r is always a 16-bit number.
3677          */
3678         r = (r & 0xFFFF) + (r >> 16);
3679
3680         r = ~r & 0xFFFF;
3681         r = r ? r : 0xFFFF;
3682
3683         *dst16_ptr = (uint16_t)r;
3684
3685         /* Thread. */
3686         thread_ip_inc(p);
3687 }
3688
3689 static inline void
3690 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3691 {
3692         struct thread *t = &p->threads[p->thread_id];
3693         struct instruction *ip = t->ip;
3694         uint8_t *dst_struct, *src_struct;
3695         uint16_t *dst16_ptr;
3696         uint32_t *src32_ptr;
3697         uint64_t r0, r1;
3698
3699         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3700
3701         /* Structs. */
3702         dst_struct = t->structs[ip->alu.dst.struct_id];
3703         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3704
3705         src_struct = t->structs[ip->alu.src.struct_id];
3706         src32_ptr = (uint32_t *)&src_struct[0];
3707
3708         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
3709         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3710         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
3711         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3712         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3713
3714         /* The first input is a 16-bit number. The second input is a 19-bit
3715          * number. Their sum is a 20-bit number.
3716          */
3717         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3718
3719         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3720          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3721          */
3722         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3723
3724         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3725          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3726          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3727          * generated, therefore the output r is always a 16-bit number.
3728          */
3729         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3730
3731         r0 = ~r0 & 0xFFFF;
3732         r0 = r0 ? r0 : 0xFFFF;
3733
3734         *dst16_ptr = (uint16_t)r0;
3735
3736         /* Thread. */
3737         thread_ip_inc(p);
3738 }
3739
3740 static inline void
3741 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3742 {
3743         struct thread *t = &p->threads[p->thread_id];
3744         struct instruction *ip = t->ip;
3745         uint8_t *dst_struct, *src_struct;
3746         uint16_t *dst16_ptr;
3747         uint32_t *src32_ptr;
3748         uint64_t r = 0;
3749         uint32_t i;
3750
3751         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3752
3753         /* Structs. */
3754         dst_struct = t->structs[ip->alu.dst.struct_id];
3755         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3756
3757         src_struct = t->structs[ip->alu.src.struct_id];
3758         src32_ptr = (uint32_t *)&src_struct[0];
3759
3760         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
3761          * Therefore, in the worst case scenario, a 35-bit number is added to a
3762          * 16-bit number (the input r), so the output r is 36-bit number.
3763          */
3764         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
3765                 r += *src32_ptr;
3766
3767         /* The first input is a 16-bit number. The second input is a 20-bit
3768          * number. Their sum is a 21-bit number.
3769          */
3770         r = (r & 0xFFFF) + (r >> 16);
3771
3772         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3773          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
3774          */
3775         r = (r & 0xFFFF) + (r >> 16);
3776
3777         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3778          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3779          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3780          * generated, therefore the output r is always a 16-bit number.
3781          */
3782         r = (r & 0xFFFF) + (r >> 16);
3783
3784         r = ~r & 0xFFFF;
3785         r = r ? r : 0xFFFF;
3786
3787         *dst16_ptr = (uint16_t)r;
3788
3789         /* Thread. */
3790         thread_ip_inc(p);
3791 }
3792
3793 #define RTE_SWX_INSTRUCTION_TOKENS_MAX 16
3794
3795 static int
3796 instr_translate(struct rte_swx_pipeline *p,
3797                 struct action *action,
3798                 char *string,
3799                 struct instruction *instr,
3800                 struct instruction_data *data)
3801 {
3802         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
3803         int n_tokens = 0, tpos = 0;
3804
3805         /* Parse the instruction string into tokens. */
3806         for ( ; ; ) {
3807                 char *token;
3808
3809                 token = strtok_r(string, " \t\v", &string);
3810                 if (!token)
3811                         break;
3812
3813                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
3814
3815                 tokens[n_tokens] = token;
3816                 n_tokens++;
3817         }
3818
3819         CHECK(n_tokens, EINVAL);
3820
3821         /* Handle the optional instruction label. */
3822         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
3823                 strcpy(data->label, tokens[0]);
3824
3825                 tpos += 2;
3826                 CHECK(n_tokens - tpos, EINVAL);
3827         }
3828
3829         /* Identify the instruction type. */
3830         if (!strcmp(tokens[tpos], "rx"))
3831                 return instr_rx_translate(p,
3832                                           action,
3833                                           &tokens[tpos],
3834                                           n_tokens - tpos,
3835                                           instr,
3836                                           data);
3837
3838         if (!strcmp(tokens[tpos], "tx"))
3839                 return instr_tx_translate(p,
3840                                           action,
3841                                           &tokens[tpos],
3842                                           n_tokens - tpos,
3843                                           instr,
3844                                           data);
3845
3846         if (!strcmp(tokens[tpos], "extract"))
3847                 return instr_hdr_extract_translate(p,
3848                                                    action,
3849                                                    &tokens[tpos],
3850                                                    n_tokens - tpos,
3851                                                    instr,
3852                                                    data);
3853
3854         if (!strcmp(tokens[tpos], "emit"))
3855                 return instr_hdr_emit_translate(p,
3856                                                 action,
3857                                                 &tokens[tpos],
3858                                                 n_tokens - tpos,
3859                                                 instr,
3860                                                 data);
3861
3862         if (!strcmp(tokens[tpos], "validate"))
3863                 return instr_hdr_validate_translate(p,
3864                                                     action,
3865                                                     &tokens[tpos],
3866                                                     n_tokens - tpos,
3867                                                     instr,
3868                                                     data);
3869
3870         if (!strcmp(tokens[tpos], "invalidate"))
3871                 return instr_hdr_invalidate_translate(p,
3872                                                       action,
3873                                                       &tokens[tpos],
3874                                                       n_tokens - tpos,
3875                                                       instr,
3876                                                       data);
3877
3878         if (!strcmp(tokens[tpos], "mov"))
3879                 return instr_mov_translate(p,
3880                                            action,
3881                                            &tokens[tpos],
3882                                            n_tokens - tpos,
3883                                            instr,
3884                                            data);
3885
3886         if (!strcmp(tokens[tpos], "dma"))
3887                 return instr_dma_translate(p,
3888                                            action,
3889                                            &tokens[tpos],
3890                                            n_tokens - tpos,
3891                                            instr,
3892                                            data);
3893
3894         if (!strcmp(tokens[tpos], "add"))
3895                 return instr_alu_add_translate(p,
3896                                                action,
3897                                                &tokens[tpos],
3898                                                n_tokens - tpos,
3899                                                instr,
3900                                                data);
3901
3902         if (!strcmp(tokens[tpos], "sub"))
3903                 return instr_alu_sub_translate(p,
3904                                                action,
3905                                                &tokens[tpos],
3906                                                n_tokens - tpos,
3907                                                instr,
3908                                                data);
3909
3910         if (!strcmp(tokens[tpos], "ckadd"))
3911                 return instr_alu_ckadd_translate(p,
3912                                                  action,
3913                                                  &tokens[tpos],
3914                                                  n_tokens - tpos,
3915                                                  instr,
3916                                                  data);
3917
3918         if (!strcmp(tokens[tpos], "cksub"))
3919                 return instr_alu_cksub_translate(p,
3920                                                  action,
3921                                                  &tokens[tpos],
3922                                                  n_tokens - tpos,
3923                                                  instr,
3924                                                  data);
3925
3926         if (!strcmp(tokens[tpos], "and"))
3927                 return instr_alu_and_translate(p,
3928                                                action,
3929                                                &tokens[tpos],
3930                                                n_tokens - tpos,
3931                                                instr,
3932                                                data);
3933
3934         if (!strcmp(tokens[tpos], "or"))
3935                 return instr_alu_or_translate(p,
3936                                               action,
3937                                               &tokens[tpos],
3938                                               n_tokens - tpos,
3939                                               instr,
3940                                               data);
3941
3942         if (!strcmp(tokens[tpos], "xor"))
3943                 return instr_alu_xor_translate(p,
3944                                                action,
3945                                                &tokens[tpos],
3946                                                n_tokens - tpos,
3947                                                instr,
3948                                                data);
3949
3950         CHECK(0, EINVAL);
3951 }
3952
3953 static uint32_t
3954 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
3955 {
3956         uint32_t count = 0, i;
3957
3958         if (!label[0])
3959                 return 0;
3960
3961         for (i = 0; i < n; i++)
3962                 if (!strcmp(label, data[i].jmp_label))
3963                         count++;
3964
3965         return count;
3966 }
3967
3968 static int
3969 instr_label_check(struct instruction_data *instruction_data,
3970                   uint32_t n_instructions)
3971 {
3972         uint32_t i;
3973
3974         /* Check that all instruction labels are unique. */
3975         for (i = 0; i < n_instructions; i++) {
3976                 struct instruction_data *data = &instruction_data[i];
3977                 char *label = data->label;
3978                 uint32_t j;
3979
3980                 if (!label[0])
3981                         continue;
3982
3983                 for (j = i + 1; j < n_instructions; j++)
3984                         CHECK(strcmp(label, data[j].label), EINVAL);
3985         }
3986
3987         /* Get users for each instruction label. */
3988         for (i = 0; i < n_instructions; i++) {
3989                 struct instruction_data *data = &instruction_data[i];
3990                 char *label = data->label;
3991
3992                 data->n_users = label_is_used(instruction_data,
3993                                               n_instructions,
3994                                               label);
3995         }
3996
3997         return 0;
3998 }
3999
4000 static int
4001 instruction_config(struct rte_swx_pipeline *p,
4002                    struct action *a,
4003                    const char **instructions,
4004                    uint32_t n_instructions)
4005 {
4006         struct instruction *instr = NULL;
4007         struct instruction_data *data = NULL;
4008         char *string = NULL;
4009         int err = 0;
4010         uint32_t i;
4011
4012         CHECK(n_instructions, EINVAL);
4013         CHECK(instructions, EINVAL);
4014         for (i = 0; i < n_instructions; i++)
4015                 CHECK(instructions[i], EINVAL);
4016
4017         /* Memory allocation. */
4018         instr = calloc(n_instructions, sizeof(struct instruction));
4019         if (!instr) {
4020                 err = ENOMEM;
4021                 goto error;
4022         }
4023
4024         data = calloc(n_instructions, sizeof(struct instruction_data));
4025         if (!data) {
4026                 err = ENOMEM;
4027                 goto error;
4028         }
4029
4030         for (i = 0; i < n_instructions; i++) {
4031                 string = strdup(instructions[i]);
4032                 if (!string) {
4033                         err = ENOMEM;
4034                         goto error;
4035                 }
4036
4037                 err = instr_translate(p, a, string, &instr[i], &data[i]);
4038                 if (err)
4039                         goto error;
4040
4041                 free(string);
4042         }
4043
4044         err = instr_label_check(data, n_instructions);
4045         if (err)
4046                 goto error;
4047
4048         free(data);
4049
4050         if (a) {
4051                 a->instructions = instr;
4052                 a->n_instructions = n_instructions;
4053         } else {
4054                 p->instructions = instr;
4055                 p->n_instructions = n_instructions;
4056         }
4057
4058         return 0;
4059
4060 error:
4061         free(string);
4062         free(data);
4063         free(instr);
4064         return err;
4065 }
4066
4067 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
4068
4069 static instr_exec_t instruction_table[] = {
4070         [INSTR_RX] = instr_rx_exec,
4071         [INSTR_TX] = instr_tx_exec,
4072
4073         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
4074         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
4075         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
4076         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
4077         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
4078         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
4079         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
4080         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
4081
4082         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
4083         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
4084         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
4085         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
4086         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
4087         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
4088         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
4089         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
4090         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
4091
4092         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
4093         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
4094
4095         [INSTR_MOV] = instr_mov_exec,
4096         [INSTR_MOV_S] = instr_mov_s_exec,
4097         [INSTR_MOV_I] = instr_mov_i_exec,
4098
4099         [INSTR_DMA_HT] = instr_dma_ht_exec,
4100         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
4101         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
4102         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
4103         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
4104         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
4105         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
4106         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
4107
4108         [INSTR_ALU_ADD] = instr_alu_add_exec,
4109         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
4110         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
4111         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
4112         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
4113         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
4114
4115         [INSTR_ALU_SUB] = instr_alu_sub_exec,
4116         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
4117         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
4118         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
4119         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
4120         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
4121
4122         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
4123         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
4124         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
4125         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
4126
4127         [INSTR_ALU_AND] = instr_alu_and_exec,
4128         [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
4129         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
4130
4131         [INSTR_ALU_OR] = instr_alu_or_exec,
4132         [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
4133         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
4134
4135         [INSTR_ALU_XOR] = instr_alu_xor_exec,
4136         [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec,
4137         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
4138 };
4139
4140 static inline void
4141 instr_exec(struct rte_swx_pipeline *p)
4142 {
4143         struct thread *t = &p->threads[p->thread_id];
4144         struct instruction *ip = t->ip;
4145         instr_exec_t instr = instruction_table[ip->type];
4146
4147         instr(p);
4148 }
4149
4150 /*
4151  * Action.
4152  */
4153 static struct action *
4154 action_find(struct rte_swx_pipeline *p, const char *name)
4155 {
4156         struct action *elem;
4157
4158         if (!name)
4159                 return NULL;
4160
4161         TAILQ_FOREACH(elem, &p->actions, node)
4162                 if (strcmp(elem->name, name) == 0)
4163                         return elem;
4164
4165         return NULL;
4166 }
4167
4168 static struct field *
4169 action_field_find(struct action *a, const char *name)
4170 {
4171         return a->st ? struct_type_field_find(a->st, name) : NULL;
4172 }
4173
4174 static struct field *
4175 action_field_parse(struct action *action, const char *name)
4176 {
4177         if (name[0] != 't' || name[1] != '.')
4178                 return NULL;
4179
4180         return action_field_find(action, &name[2]);
4181 }
4182
4183 int
4184 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
4185                                const char *name,
4186                                const char *args_struct_type_name,
4187                                const char **instructions,
4188                                uint32_t n_instructions)
4189 {
4190         struct struct_type *args_struct_type;
4191         struct action *a;
4192         int err;
4193
4194         CHECK(p, EINVAL);
4195
4196         CHECK_NAME(name, EINVAL);
4197         CHECK(!action_find(p, name), EEXIST);
4198
4199         if (args_struct_type_name) {
4200                 CHECK_NAME(args_struct_type_name, EINVAL);
4201                 args_struct_type = struct_type_find(p, args_struct_type_name);
4202                 CHECK(args_struct_type, EINVAL);
4203         } else {
4204                 args_struct_type = NULL;
4205         }
4206
4207         /* Node allocation. */
4208         a = calloc(1, sizeof(struct action));
4209         CHECK(a, ENOMEM);
4210
4211         /* Node initialization. */
4212         strcpy(a->name, name);
4213         a->st = args_struct_type;
4214         a->id = p->n_actions;
4215
4216         /* Instruction translation. */
4217         err = instruction_config(p, a, instructions, n_instructions);
4218         if (err) {
4219                 free(a);
4220                 return err;
4221         }
4222
4223         /* Node add to tailq. */
4224         TAILQ_INSERT_TAIL(&p->actions, a, node);
4225         p->n_actions++;
4226
4227         return 0;
4228 }
4229
4230 static int
4231 action_build(struct rte_swx_pipeline *p)
4232 {
4233         struct action *action;
4234
4235         p->action_instructions = calloc(p->n_actions,
4236                                         sizeof(struct instruction *));
4237         CHECK(p->action_instructions, ENOMEM);
4238
4239         TAILQ_FOREACH(action, &p->actions, node)
4240                 p->action_instructions[action->id] = action->instructions;
4241
4242         return 0;
4243 }
4244
4245 static void
4246 action_build_free(struct rte_swx_pipeline *p)
4247 {
4248         free(p->action_instructions);
4249         p->action_instructions = NULL;
4250 }
4251
4252 static void
4253 action_free(struct rte_swx_pipeline *p)
4254 {
4255         action_build_free(p);
4256
4257         for ( ; ; ) {
4258                 struct action *action;
4259
4260                 action = TAILQ_FIRST(&p->actions);
4261                 if (!action)
4262                         break;
4263
4264                 TAILQ_REMOVE(&p->actions, action, node);
4265                 free(action->instructions);
4266                 free(action);
4267         }
4268 }
4269
4270 /*
4271  * Table.
4272  */
4273 static struct table_type *
4274 table_type_find(struct rte_swx_pipeline *p, const char *name)
4275 {
4276         struct table_type *elem;
4277
4278         TAILQ_FOREACH(elem, &p->table_types, node)
4279                 if (strcmp(elem->name, name) == 0)
4280                         return elem;
4281
4282         return NULL;
4283 }
4284
4285 static struct table_type *
4286 table_type_resolve(struct rte_swx_pipeline *p,
4287                    const char *recommended_type_name,
4288                    enum rte_swx_table_match_type match_type)
4289 {
4290         struct table_type *elem;
4291
4292         /* Only consider the recommended type if the match type is correct. */
4293         if (recommended_type_name)
4294                 TAILQ_FOREACH(elem, &p->table_types, node)
4295                         if (!strcmp(elem->name, recommended_type_name) &&
4296                             (elem->match_type == match_type))
4297                                 return elem;
4298
4299         /* Ignore the recommended type and get the first element with this match
4300          * type.
4301          */
4302         TAILQ_FOREACH(elem, &p->table_types, node)
4303                 if (elem->match_type == match_type)
4304                         return elem;
4305
4306         return NULL;
4307 }
4308
4309 static struct table *
4310 table_find(struct rte_swx_pipeline *p, const char *name)
4311 {
4312         struct table *elem;
4313
4314         TAILQ_FOREACH(elem, &p->tables, node)
4315                 if (strcmp(elem->name, name) == 0)
4316                         return elem;
4317
4318         return NULL;
4319 }
4320
4321 static struct table *
4322 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
4323 {
4324         struct table *table = NULL;
4325
4326         TAILQ_FOREACH(table, &p->tables, node)
4327                 if (table->id == id)
4328                         return table;
4329
4330         return NULL;
4331 }
4332
4333 int
4334 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
4335                                      const char *name,
4336                                      enum rte_swx_table_match_type match_type,
4337                                      struct rte_swx_table_ops *ops)
4338 {
4339         struct table_type *elem;
4340
4341         CHECK(p, EINVAL);
4342
4343         CHECK_NAME(name, EINVAL);
4344         CHECK(!table_type_find(p, name), EEXIST);
4345
4346         CHECK(ops, EINVAL);
4347         CHECK(ops->create, EINVAL);
4348         CHECK(ops->lkp, EINVAL);
4349         CHECK(ops->free, EINVAL);
4350
4351         /* Node allocation. */
4352         elem = calloc(1, sizeof(struct table_type));
4353         CHECK(elem, ENOMEM);
4354
4355         /* Node initialization. */
4356         strcpy(elem->name, name);
4357         elem->match_type = match_type;
4358         memcpy(&elem->ops, ops, sizeof(*ops));
4359
4360         /* Node add to tailq. */
4361         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
4362
4363         return 0;
4364 }
4365
4366 static enum rte_swx_table_match_type
4367 table_match_type_resolve(struct rte_swx_match_field_params *fields,
4368                          uint32_t n_fields)
4369 {
4370         uint32_t i;
4371
4372         for (i = 0; i < n_fields; i++)
4373                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
4374                         break;
4375
4376         if (i == n_fields)
4377                 return RTE_SWX_TABLE_MATCH_EXACT;
4378
4379         if ((i == n_fields - 1) &&
4380             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
4381                 return RTE_SWX_TABLE_MATCH_LPM;
4382
4383         return RTE_SWX_TABLE_MATCH_WILDCARD;
4384 }
4385
4386 int
4387 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
4388                               const char *name,
4389                               struct rte_swx_pipeline_table_params *params,
4390                               const char *recommended_table_type_name,
4391                               const char *args,
4392                               uint32_t size)
4393 {
4394         struct table_type *type;
4395         struct table *t;
4396         struct action *default_action;
4397         struct header *header = NULL;
4398         int is_header = 0;
4399         uint32_t offset_prev = 0, action_data_size_max = 0, i;
4400
4401         CHECK(p, EINVAL);
4402
4403         CHECK_NAME(name, EINVAL);
4404         CHECK(!table_find(p, name), EEXIST);
4405
4406         CHECK(params, EINVAL);
4407
4408         /* Match checks. */
4409         CHECK(!params->n_fields || params->fields, EINVAL);
4410         for (i = 0; i < params->n_fields; i++) {
4411                 struct rte_swx_match_field_params *field = &params->fields[i];
4412                 struct header *h;
4413                 struct field *hf, *mf;
4414                 uint32_t offset;
4415
4416                 CHECK_NAME(field->name, EINVAL);
4417
4418                 hf = header_field_parse(p, field->name, &h);
4419                 mf = metadata_field_parse(p, field->name);
4420                 CHECK(hf || mf, EINVAL);
4421
4422                 offset = hf ? hf->offset : mf->offset;
4423
4424                 if (i == 0) {
4425                         is_header = hf ? 1 : 0;
4426                         header = hf ? h : NULL;
4427                         offset_prev = offset;
4428
4429                         continue;
4430                 }
4431
4432                 CHECK((is_header && hf && (h->id == header->id)) ||
4433                       (!is_header && mf), EINVAL);
4434
4435                 CHECK(offset > offset_prev, EINVAL);
4436                 offset_prev = offset;
4437         }
4438
4439         /* Action checks. */
4440         CHECK(params->n_actions, EINVAL);
4441         CHECK(params->action_names, EINVAL);
4442         for (i = 0; i < params->n_actions; i++) {
4443                 const char *action_name = params->action_names[i];
4444                 struct action *a;
4445                 uint32_t action_data_size;
4446
4447                 CHECK(action_name, EINVAL);
4448
4449                 a = action_find(p, action_name);
4450                 CHECK(a, EINVAL);
4451
4452                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
4453                 if (action_data_size > action_data_size_max)
4454                         action_data_size_max = action_data_size;
4455         }
4456
4457         CHECK(params->default_action_name, EINVAL);
4458         for (i = 0; i < p->n_actions; i++)
4459                 if (!strcmp(params->action_names[i],
4460                             params->default_action_name))
4461                         break;
4462         CHECK(i < params->n_actions, EINVAL);
4463         default_action = action_find(p, params->default_action_name);
4464         CHECK((default_action->st && params->default_action_data) ||
4465               !params->default_action_data, EINVAL);
4466
4467         /* Table type checks. */
4468         if (params->n_fields) {
4469                 enum rte_swx_table_match_type match_type;
4470
4471                 match_type = table_match_type_resolve(params->fields,
4472                                                       params->n_fields);
4473                 type = table_type_resolve(p,
4474                                           recommended_table_type_name,
4475                                           match_type);
4476                 CHECK(type, EINVAL);
4477         } else {
4478                 type = NULL;
4479         }
4480
4481         /* Memory allocation. */
4482         t = calloc(1, sizeof(struct table));
4483         CHECK(t, ENOMEM);
4484
4485         t->fields = calloc(params->n_fields, sizeof(struct match_field));
4486         if (!t->fields) {
4487                 free(t);
4488                 CHECK(0, ENOMEM);
4489         }
4490
4491         t->actions = calloc(params->n_actions, sizeof(struct action *));
4492         if (!t->actions) {
4493                 free(t->fields);
4494                 free(t);
4495                 CHECK(0, ENOMEM);
4496         }
4497
4498         if (action_data_size_max) {
4499                 t->default_action_data = calloc(1, action_data_size_max);
4500                 if (!t->default_action_data) {
4501                         free(t->actions);
4502                         free(t->fields);
4503                         free(t);
4504                         CHECK(0, ENOMEM);
4505                 }
4506         }
4507
4508         /* Node initialization. */
4509         strcpy(t->name, name);
4510         if (args && args[0])
4511                 strcpy(t->args, args);
4512         t->type = type;
4513
4514         for (i = 0; i < params->n_fields; i++) {
4515                 struct rte_swx_match_field_params *field = &params->fields[i];
4516                 struct match_field *f = &t->fields[i];
4517
4518                 f->match_type = field->match_type;
4519                 f->field = is_header ?
4520                         header_field_parse(p, field->name, NULL) :
4521                         metadata_field_parse(p, field->name);
4522         }
4523         t->n_fields = params->n_fields;
4524         t->is_header = is_header;
4525         t->header = header;
4526
4527         for (i = 0; i < params->n_actions; i++)
4528                 t->actions[i] = action_find(p, params->action_names[i]);
4529         t->default_action = default_action;
4530         if (default_action->st)
4531                 memcpy(t->default_action_data,
4532                        params->default_action_data,
4533                        default_action->st->n_bits / 8);
4534         t->n_actions = params->n_actions;
4535         t->default_action_is_const = params->default_action_is_const;
4536         t->action_data_size_max = action_data_size_max;
4537
4538         t->size = size;
4539         t->id = p->n_tables;
4540
4541         /* Node add to tailq. */
4542         TAILQ_INSERT_TAIL(&p->tables, t, node);
4543         p->n_tables++;
4544
4545         return 0;
4546 }
4547
4548 static struct rte_swx_table_params *
4549 table_params_get(struct table *table)
4550 {
4551         struct rte_swx_table_params *params;
4552         struct field *first, *last;
4553         uint8_t *key_mask;
4554         uint32_t key_size, key_offset, action_data_size, i;
4555
4556         /* Memory allocation. */
4557         params = calloc(1, sizeof(struct rte_swx_table_params));
4558         if (!params)
4559                 return NULL;
4560
4561         /* Key offset and size. */
4562         first = table->fields[0].field;
4563         last = table->fields[table->n_fields - 1].field;
4564         key_offset = first->offset / 8;
4565         key_size = (last->offset + last->n_bits - first->offset) / 8;
4566
4567         /* Memory allocation. */
4568         key_mask = calloc(1, key_size);
4569         if (!key_mask) {
4570                 free(params);
4571                 return NULL;
4572         }
4573
4574         /* Key mask. */
4575         for (i = 0; i < table->n_fields; i++) {
4576                 struct field *f = table->fields[i].field;
4577                 uint32_t start = (f->offset - first->offset) / 8;
4578                 size_t size = f->n_bits / 8;
4579
4580                 memset(&key_mask[start], 0xFF, size);
4581         }
4582
4583         /* Action data size. */
4584         action_data_size = 0;
4585         for (i = 0; i < table->n_actions; i++) {
4586                 struct action *action = table->actions[i];
4587                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
4588
4589                 if (ads > action_data_size)
4590                         action_data_size = ads;
4591         }
4592
4593         /* Fill in. */
4594         params->match_type = table->type->match_type;
4595         params->key_size = key_size;
4596         params->key_offset = key_offset;
4597         params->key_mask0 = key_mask;
4598         params->action_data_size = action_data_size;
4599         params->n_keys_max = table->size;
4600
4601         return params;
4602 }
4603
4604 static void
4605 table_params_free(struct rte_swx_table_params *params)
4606 {
4607         if (!params)
4608                 return;
4609
4610         free(params->key_mask0);
4611         free(params);
4612 }
4613
4614 static int
4615 table_state_build(struct rte_swx_pipeline *p)
4616 {
4617         struct table *table;
4618
4619         p->table_state = calloc(p->n_tables,
4620                                 sizeof(struct rte_swx_table_state));
4621         CHECK(p->table_state, ENOMEM);
4622
4623         TAILQ_FOREACH(table, &p->tables, node) {
4624                 struct rte_swx_table_state *ts = &p->table_state[table->id];
4625
4626                 if (table->type) {
4627                         struct rte_swx_table_params *params;
4628
4629                         /* ts->obj. */
4630                         params = table_params_get(table);
4631                         CHECK(params, ENOMEM);
4632
4633                         ts->obj = table->type->ops.create(params,
4634                                 NULL,
4635                                 table->args,
4636                                 p->numa_node);
4637
4638                         table_params_free(params);
4639                         CHECK(ts->obj, ENODEV);
4640                 }
4641
4642                 /* ts->default_action_data. */
4643                 if (table->action_data_size_max) {
4644                         ts->default_action_data =
4645                                 malloc(table->action_data_size_max);
4646                         CHECK(ts->default_action_data, ENOMEM);
4647
4648                         memcpy(ts->default_action_data,
4649                                table->default_action_data,
4650                                table->action_data_size_max);
4651                 }
4652
4653                 /* ts->default_action_id. */
4654                 ts->default_action_id = table->default_action->id;
4655         }
4656
4657         return 0;
4658 }
4659
4660 static void
4661 table_state_build_free(struct rte_swx_pipeline *p)
4662 {
4663         uint32_t i;
4664
4665         if (!p->table_state)
4666                 return;
4667
4668         for (i = 0; i < p->n_tables; i++) {
4669                 struct rte_swx_table_state *ts = &p->table_state[i];
4670                 struct table *table = table_find_by_id(p, i);
4671
4672                 /* ts->obj. */
4673                 if (table->type && ts->obj)
4674                         table->type->ops.free(ts->obj);
4675
4676                 /* ts->default_action_data. */
4677                 free(ts->default_action_data);
4678         }
4679
4680         free(p->table_state);
4681         p->table_state = NULL;
4682 }
4683
4684 static void
4685 table_state_free(struct rte_swx_pipeline *p)
4686 {
4687         table_state_build_free(p);
4688 }
4689
4690 static int
4691 table_stub_lkp(void *table __rte_unused,
4692                void *mailbox __rte_unused,
4693                uint8_t **key __rte_unused,
4694                uint64_t *action_id __rte_unused,
4695                uint8_t **action_data __rte_unused,
4696                int *hit)
4697 {
4698         *hit = 0;
4699         return 1; /* DONE. */
4700 }
4701
4702 static int
4703 table_build(struct rte_swx_pipeline *p)
4704 {
4705         uint32_t i;
4706
4707         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
4708                 struct thread *t = &p->threads[i];
4709                 struct table *table;
4710
4711                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
4712                 CHECK(t->tables, ENOMEM);
4713
4714                 TAILQ_FOREACH(table, &p->tables, node) {
4715                         struct table_runtime *r = &t->tables[table->id];
4716
4717                         if (table->type) {
4718                                 uint64_t size;
4719
4720                                 size = table->type->ops.mailbox_size_get();
4721
4722                                 /* r->func. */
4723                                 r->func = table->type->ops.lkp;
4724
4725                                 /* r->mailbox. */
4726                                 if (size) {
4727                                         r->mailbox = calloc(1, size);
4728                                         CHECK(r->mailbox, ENOMEM);
4729                                 }
4730
4731                                 /* r->key. */
4732                                 r->key = table->is_header ?
4733                                         &t->structs[table->header->struct_id] :
4734                                         &t->structs[p->metadata_struct_id];
4735                         } else {
4736                                 r->func = table_stub_lkp;
4737                         }
4738                 }
4739         }
4740
4741         return 0;
4742 }
4743
4744 static void
4745 table_build_free(struct rte_swx_pipeline *p)
4746 {
4747         uint32_t i;
4748
4749         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
4750                 struct thread *t = &p->threads[i];
4751                 uint32_t j;
4752
4753                 if (!t->tables)
4754                         continue;
4755
4756                 for (j = 0; j < p->n_tables; j++) {
4757                         struct table_runtime *r = &t->tables[j];
4758
4759                         free(r->mailbox);
4760                 }
4761
4762                 free(t->tables);
4763                 t->tables = NULL;
4764         }
4765 }
4766
4767 static void
4768 table_free(struct rte_swx_pipeline *p)
4769 {
4770         table_build_free(p);
4771
4772         /* Tables. */
4773         for ( ; ; ) {
4774                 struct table *elem;
4775
4776                 elem = TAILQ_FIRST(&p->tables);
4777                 if (!elem)
4778                         break;
4779
4780                 TAILQ_REMOVE(&p->tables, elem, node);
4781                 free(elem->fields);
4782                 free(elem->actions);
4783                 free(elem->default_action_data);
4784                 free(elem);
4785         }
4786
4787         /* Table types. */
4788         for ( ; ; ) {
4789                 struct table_type *elem;
4790
4791                 elem = TAILQ_FIRST(&p->table_types);
4792                 if (!elem)
4793                         break;
4794
4795                 TAILQ_REMOVE(&p->table_types, elem, node);
4796                 free(elem);
4797         }
4798 }
4799
4800 /*
4801  * Pipeline.
4802  */
4803 int
4804 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
4805 {
4806         struct rte_swx_pipeline *pipeline;
4807
4808         /* Check input parameters. */
4809         CHECK(p, EINVAL);
4810
4811         /* Memory allocation. */
4812         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
4813         CHECK(pipeline, ENOMEM);
4814
4815         /* Initialization. */
4816         TAILQ_INIT(&pipeline->struct_types);
4817         TAILQ_INIT(&pipeline->port_in_types);
4818         TAILQ_INIT(&pipeline->ports_in);
4819         TAILQ_INIT(&pipeline->port_out_types);
4820         TAILQ_INIT(&pipeline->ports_out);
4821         TAILQ_INIT(&pipeline->extern_types);
4822         TAILQ_INIT(&pipeline->extern_objs);
4823         TAILQ_INIT(&pipeline->extern_funcs);
4824         TAILQ_INIT(&pipeline->headers);
4825         TAILQ_INIT(&pipeline->actions);
4826         TAILQ_INIT(&pipeline->table_types);
4827         TAILQ_INIT(&pipeline->tables);
4828
4829         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
4830         pipeline->numa_node = numa_node;
4831
4832         *p = pipeline;
4833         return 0;
4834 }
4835
4836 void
4837 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
4838 {
4839         if (!p)
4840                 return;
4841
4842         free(p->instructions);
4843
4844         table_state_free(p);
4845         table_free(p);
4846         action_free(p);
4847         metadata_free(p);
4848         header_free(p);
4849         extern_func_free(p);
4850         extern_obj_free(p);
4851         port_out_free(p);
4852         port_in_free(p);
4853         struct_free(p);
4854
4855         free(p);
4856 }
4857
4858 int
4859 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
4860                                      const char **instructions,
4861                                      uint32_t n_instructions)
4862 {
4863         int err;
4864         uint32_t i;
4865
4866         err = instruction_config(p, NULL, instructions, n_instructions);
4867         if (err)
4868                 return err;
4869
4870         /* Thread instruction pointer reset. */
4871         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
4872                 struct thread *t = &p->threads[i];
4873
4874                 thread_ip_reset(p, t);
4875         }
4876
4877         return 0;
4878 }
4879
4880 int
4881 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
4882 {
4883         int status;
4884
4885         CHECK(p, EINVAL);
4886         CHECK(p->build_done == 0, EEXIST);
4887
4888         status = port_in_build(p);
4889         if (status)
4890                 goto error;
4891
4892         status = port_out_build(p);
4893         if (status)
4894                 goto error;
4895
4896         status = struct_build(p);
4897         if (status)
4898                 goto error;
4899
4900         status = extern_obj_build(p);
4901         if (status)
4902                 goto error;
4903
4904         status = extern_func_build(p);
4905         if (status)
4906                 goto error;
4907
4908         status = header_build(p);
4909         if (status)
4910                 goto error;
4911
4912         status = metadata_build(p);
4913         if (status)
4914                 goto error;
4915
4916         status = action_build(p);
4917         if (status)
4918                 goto error;
4919
4920         status = table_build(p);
4921         if (status)
4922                 goto error;
4923
4924         status = table_state_build(p);
4925         if (status)
4926                 goto error;
4927
4928         p->build_done = 1;
4929         return 0;
4930
4931 error:
4932         table_state_build_free(p);
4933         table_build_free(p);
4934         action_build_free(p);
4935         metadata_build_free(p);
4936         header_build_free(p);
4937         extern_func_build_free(p);
4938         extern_obj_build_free(p);
4939         port_out_build_free(p);
4940         port_in_build_free(p);
4941         struct_build_free(p);
4942
4943         return status;
4944 }
4945
4946 void
4947 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
4948 {
4949         uint32_t i;
4950
4951         for (i = 0; i < n_instructions; i++)
4952                 instr_exec(p);
4953 }
4954
4955 /*
4956  * Control.
4957  */
4958 int
4959 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
4960                                  struct rte_swx_table_state **table_state)
4961 {
4962         if (!p || !table_state || !p->build_done)
4963                 return -EINVAL;
4964
4965         *table_state = p->table_state;
4966         return 0;
4967 }
4968
4969 int
4970 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
4971                                  struct rte_swx_table_state *table_state)
4972 {
4973         if (!p || !table_state || !p->build_done)
4974                 return -EINVAL;
4975
4976         p->table_state = table_state;
4977         return 0;
4978 }