lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9 #include <arpa/inet.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_prefetch.h>
  13 #include <rte_byteorder.h>
  14
  15 #include "rte_swx_pipeline.h"
  16 #include "rte_swx_ctl.h"
  17
  18 #define CHECK(condition, err_code)                                             \
  19 do {                                                                           \
  20         if (!(condition))                                                      \
  21                 return -(err_code);                                            \
  22 } while (0)
  23
  24 #define CHECK_NAME(name, err_code)                                             \
  25         CHECK((name) && (name)[0], err_code)
  26
  27 #ifndef TRACE_LEVEL
  28 #define TRACE_LEVEL 0
  29 #endif
  30
  31 #if TRACE_LEVEL
  32 #define TRACE(...) printf(__VA_ARGS__)
  33 #else
  34 #define TRACE(...)
  35 #endif
  36
  37 #define ntoh64(x) rte_be_to_cpu_64(x)
  38 #define hton64(x) rte_cpu_to_be_64(x)
  39
  40 /*
  41  * Struct.
  42  */
  43 struct field {
  44         char name[RTE_SWX_NAME_SIZE];
  45         uint32_t n_bits;
  46         uint32_t offset;
  47 };
  48
  49 struct struct_type {
  50         TAILQ_ENTRY(struct_type) node;
  51         char name[RTE_SWX_NAME_SIZE];
  52         struct field *fields;
  53         uint32_t n_fields;
  54         uint32_t n_bits;
  55 };
  56
  57 TAILQ_HEAD(struct_type_tailq, struct_type);
  58
  59 /*
  60  * Input port.
  61  */
  62 struct port_in_type {
  63         TAILQ_ENTRY(port_in_type) node;
  64         char name[RTE_SWX_NAME_SIZE];
  65         struct rte_swx_port_in_ops ops;
  66 };
  67
  68 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  69
  70 struct port_in {
  71         TAILQ_ENTRY(port_in) node;
  72         struct port_in_type *type;
  73         void *obj;
  74         uint32_t id;
  75 };
  76
  77 TAILQ_HEAD(port_in_tailq, port_in);
  78
  79 struct port_in_runtime {
  80         rte_swx_port_in_pkt_rx_t pkt_rx;
  81         void *obj;
  82 };
  83
  84 /*
  85  * Output port.
  86  */
  87 struct port_out_type {
  88         TAILQ_ENTRY(port_out_type) node;
  89         char name[RTE_SWX_NAME_SIZE];
  90         struct rte_swx_port_out_ops ops;
  91 };
  92
  93 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  94
  95 struct port_out {
  96         TAILQ_ENTRY(port_out) node;
  97         struct port_out_type *type;
  98         void *obj;
  99         uint32_t id;
 100 };
 101
 102 TAILQ_HEAD(port_out_tailq, port_out);
 103
 104 struct port_out_runtime {
 105         rte_swx_port_out_pkt_tx_t pkt_tx;
 106         rte_swx_port_out_flush_t flush;
 107         void *obj;
 108 };
 109
 110 /*
 111  * Extern object.
 112  */
 113 struct extern_type_member_func {
 114         TAILQ_ENTRY(extern_type_member_func) node;
 115         char name[RTE_SWX_NAME_SIZE];
 116         rte_swx_extern_type_member_func_t func;
 117         uint32_t id;
 118 };
 119
 120 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 121
 122 struct extern_type {
 123         TAILQ_ENTRY(extern_type) node;
 124         char name[RTE_SWX_NAME_SIZE];
 125         struct struct_type *mailbox_struct_type;
 126         rte_swx_extern_type_constructor_t constructor;
 127         rte_swx_extern_type_destructor_t destructor;
 128         struct extern_type_member_func_tailq funcs;
 129         uint32_t n_funcs;
 130 };
 131
 132 TAILQ_HEAD(extern_type_tailq, extern_type);
 133
 134 struct extern_obj {
 135         TAILQ_ENTRY(extern_obj) node;
 136         char name[RTE_SWX_NAME_SIZE];
 137         struct extern_type *type;
 138         void *obj;
 139         uint32_t struct_id;
 140         uint32_t id;
 141 };
 142
 143 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 144
 145 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 146 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 147 #endif
 148
 149 struct extern_obj_runtime {
 150         void *obj;
 151         uint8_t *mailbox;
 152         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 153 };
 154
 155 /*
 156  * Extern function.
 157  */
 158 struct extern_func {
 159         TAILQ_ENTRY(extern_func) node;
 160         char name[RTE_SWX_NAME_SIZE];
 161         struct struct_type *mailbox_struct_type;
 162         rte_swx_extern_func_t func;
 163         uint32_t struct_id;
 164         uint32_t id;
 165 };
 166
 167 TAILQ_HEAD(extern_func_tailq, extern_func);
 168
 169 struct extern_func_runtime {
 170         uint8_t *mailbox;
 171         rte_swx_extern_func_t func;
 172 };
 173
 174 /*
 175  * Header.
 176  */
 177 struct header {
 178         TAILQ_ENTRY(header) node;
 179         char name[RTE_SWX_NAME_SIZE];
 180         struct struct_type *st;
 181         uint32_t struct_id;
 182         uint32_t id;
 183 };
 184
 185 TAILQ_HEAD(header_tailq, header);
 186
 187 struct header_runtime {
 188         uint8_t *ptr0;
 189 };
 190
 191 struct header_out_runtime {
 192         uint8_t *ptr0;
 193         uint8_t *ptr;
 194         uint32_t n_bytes;
 195 };
 196
 197 /*
 198  * Instruction.
 199  */
 200
 201 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 202  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 203  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 204  * when transferred to packet meta-data and in NBO when transferred to packet
 205  * headers.
 206  */
 207
 208 /* Notation conventions:
 209  *    -Header field: H = h.header.field (dst/src)
 210  *    -Meta-data field: M = m.field (dst/src)
 211  *    -Extern object mailbox field: E = e.field (dst/src)
 212  *    -Extern function mailbox field: F = f.field (dst/src)
 213  *    -Table action data field: T = t.field (src only)
 214  *    -Immediate value: I = 32-bit unsigned value (src only)
 215  */
 216
 217 enum instruction_type {
 218         /* rx m.port_in */
 219         INSTR_RX,
 220
 221         /* tx m.port_out */
 222         INSTR_TX,
 223
 224         /* extract h.header */
 225         INSTR_HDR_EXTRACT,
 226         INSTR_HDR_EXTRACT2,
 227         INSTR_HDR_EXTRACT3,
 228         INSTR_HDR_EXTRACT4,
 229         INSTR_HDR_EXTRACT5,
 230         INSTR_HDR_EXTRACT6,
 231         INSTR_HDR_EXTRACT7,
 232         INSTR_HDR_EXTRACT8,
 233
 234         /* emit h.header */
 235         INSTR_HDR_EMIT,
 236         INSTR_HDR_EMIT_TX,
 237         INSTR_HDR_EMIT2_TX,
 238         INSTR_HDR_EMIT3_TX,
 239         INSTR_HDR_EMIT4_TX,
 240         INSTR_HDR_EMIT5_TX,
 241         INSTR_HDR_EMIT6_TX,
 242         INSTR_HDR_EMIT7_TX,
 243         INSTR_HDR_EMIT8_TX,
 244
 245         /* validate h.header */
 246         INSTR_HDR_VALIDATE,
 247
 248         /* invalidate h.header */
 249         INSTR_HDR_INVALIDATE,
 250
 251         /* mov dst src
 252          * dst = src
 253          * dst = HMEF, src = HMEFTI
 254          */
 255         INSTR_MOV,   /* dst = MEF, src = MEFT */
 256         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 257         INSTR_MOV_I, /* dst = HMEF, src = I */
 258
 259         /* dma h.header t.field
 260          * memcpy(h.header, t.field, sizeof(h.header))
 261          */
 262         INSTR_DMA_HT,
 263         INSTR_DMA_HT2,
 264         INSTR_DMA_HT3,
 265         INSTR_DMA_HT4,
 266         INSTR_DMA_HT5,
 267         INSTR_DMA_HT6,
 268         INSTR_DMA_HT7,
 269         INSTR_DMA_HT8,
 270
 271         /* add dst src
 272          * dst += src
 273          * dst = HMEF, src = HMEFTI
 274          */
 275         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 276         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 277         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 278         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 279         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 280         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 281
 282         /* sub dst src
 283          * dst -= src
 284          * dst = HMEF, src = HMEFTI
 285          */
 286         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 287         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 288         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 289         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 290         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 291         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 292 };
 293
 294 struct instr_operand {
 295         uint8_t struct_id;
 296         uint8_t n_bits;
 297         uint8_t offset;
 298         uint8_t pad;
 299 };
 300
 301 struct instr_io {
 302         struct {
 303                 uint8_t offset;
 304                 uint8_t n_bits;
 305                 uint8_t pad[2];
 306         } io;
 307
 308         struct {
 309                 uint8_t header_id[8];
 310                 uint8_t struct_id[8];
 311                 uint8_t n_bytes[8];
 312         } hdr;
 313 };
 314
 315 struct instr_hdr_validity {
 316         uint8_t header_id;
 317 };
 318
 319 struct instr_dst_src {
 320         struct instr_operand dst;
 321         union {
 322                 struct instr_operand src;
 323                 uint32_t src_val;
 324         };
 325 };
 326
 327 struct instr_dma {
 328         struct {
 329                 uint8_t header_id[8];
 330                 uint8_t struct_id[8];
 331         } dst;
 332
 333         struct {
 334                 uint8_t offset[8];
 335         } src;
 336
 337         uint16_t n_bytes[8];
 338 };
 339
 340 struct instruction {
 341         enum instruction_type type;
 342         union {
 343                 struct instr_io io;
 344                 struct instr_hdr_validity valid;
 345                 struct instr_dst_src mov;
 346                 struct instr_dma dma;
 347                 struct instr_dst_src alu;
 348         };
 349 };
 350
 351 struct instruction_data {
 352         char label[RTE_SWX_NAME_SIZE];
 353         char jmp_label[RTE_SWX_NAME_SIZE];
 354         uint32_t n_users; /* user = jmp instruction to this instruction. */
 355         int invalid;
 356 };
 357
 358 /*
 359  * Action.
 360  */
 361 struct action {
 362         TAILQ_ENTRY(action) node;
 363         char name[RTE_SWX_NAME_SIZE];
 364         struct struct_type *st;
 365         struct instruction *instructions;
 366         uint32_t n_instructions;
 367         uint32_t id;
 368 };
 369
 370 TAILQ_HEAD(action_tailq, action);
 371
 372 /*
 373  * Table.
 374  */
 375 struct table_type {
 376         TAILQ_ENTRY(table_type) node;
 377         char name[RTE_SWX_NAME_SIZE];
 378         enum rte_swx_table_match_type match_type;
 379         struct rte_swx_table_ops ops;
 380 };
 381
 382 TAILQ_HEAD(table_type_tailq, table_type);
 383
 384 struct match_field {
 385         enum rte_swx_table_match_type match_type;
 386         struct field *field;
 387 };
 388
 389 struct table {
 390         TAILQ_ENTRY(table) node;
 391         char name[RTE_SWX_NAME_SIZE];
 392         char args[RTE_SWX_NAME_SIZE];
 393         struct table_type *type; /* NULL when n_fields == 0. */
 394
 395         /* Match. */
 396         struct match_field *fields;
 397         uint32_t n_fields;
 398         int is_header; /* Only valid when n_fields > 0. */
 399         struct header *header; /* Only valid when n_fields > 0. */
 400
 401         /* Action. */
 402         struct action **actions;
 403         struct action *default_action;
 404         uint8_t *default_action_data;
 405         uint32_t n_actions;
 406         int default_action_is_const;
 407         uint32_t action_data_size_max;
 408
 409         uint32_t size;
 410         uint32_t id;
 411 };
 412
 413 TAILQ_HEAD(table_tailq, table);
 414
 415 struct table_runtime {
 416         rte_swx_table_lookup_t func;
 417         void *mailbox;
 418         uint8_t **key;
 419 };
 420
 421 /*
 422  * Pipeline.
 423  */
 424 struct thread {
 425         /* Packet. */
 426         struct rte_swx_pkt pkt;
 427         uint8_t *ptr;
 428
 429         /* Structures. */
 430         uint8_t **structs;
 431
 432         /* Packet headers. */
 433         struct header_runtime *headers; /* Extracted or generated headers. */
 434         struct header_out_runtime *headers_out; /* Emitted headers. */
 435         uint8_t *header_storage;
 436         uint8_t *header_out_storage;
 437         uint64_t valid_headers;
 438         uint32_t n_headers_out;
 439
 440         /* Packet meta-data. */
 441         uint8_t *metadata;
 442
 443         /* Tables. */
 444         struct table_runtime *tables;
 445         struct rte_swx_table_state *table_state;
 446         uint64_t action_id;
 447         int hit; /* 0 = Miss, 1 = Hit. */
 448
 449         /* Extern objects and functions. */
 450         struct extern_obj_runtime *extern_objs;
 451         struct extern_func_runtime *extern_funcs;
 452
 453         /* Instructions. */
 454         struct instruction *ip;
 455         struct instruction *ret;
 456 };
 457
 458 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 459 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 460 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 461
 462 #define ALU(thread, ip, operator)  \
 463 {                                                                              \
 464         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 465         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 466         uint64_t dst64 = *dst64_ptr;                                           \
 467         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 468         uint64_t dst = dst64 & dst64_mask;                                     \
 469                                                                                \
 470         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 471         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 472         uint64_t src64 = *src64_ptr;                                           \
 473         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 474         uint64_t src = src64 & src64_mask;                                     \
 475                                                                                \
 476         uint64_t result = dst operator src;                                    \
 477                                                                                \
 478         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 479 }
 480
 481 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 482
 483 #define ALU_S(thread, ip, operator)  \
 484 {                                                                              \
 485         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 486         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 487         uint64_t dst64 = *dst64_ptr;                                           \
 488         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 489         uint64_t dst = dst64 & dst64_mask;                                     \
 490                                                                                \
 491         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 492         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 493         uint64_t src64 = *src64_ptr;                                           \
 494         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 495                                                                                \
 496         uint64_t result = dst operator src;                                    \
 497                                                                                \
 498         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 499 }
 500
 501 #define ALU_MH ALU_S
 502
 503 #define ALU_HM(thread, ip, operator)  \
 504 {                                                                              \
 505         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 506         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 507         uint64_t dst64 = *dst64_ptr;                                           \
 508         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 509         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 510                                                                                \
 511         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 512         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 513         uint64_t src64 = *src64_ptr;                                           \
 514         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 515         uint64_t src = src64 & src64_mask;                                     \
 516                                                                                \
 517         uint64_t result = dst operator src;                                    \
 518         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 519                                                                                \
 520         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 521 }
 522
 523 #define ALU_HH(thread, ip, operator)  \
 524 {                                                                              \
 525         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 526         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 527         uint64_t dst64 = *dst64_ptr;                                           \
 528         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 529         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 530                                                                                \
 531         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 532         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 533         uint64_t src64 = *src64_ptr;                                           \
 534         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 535                                                                                \
 536         uint64_t result = dst operator src;                                    \
 537         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 538                                                                                \
 539         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 540 }
 541
 542 #else
 543
 544 #define ALU_S ALU
 545 #define ALU_MH ALU
 546 #define ALU_HM ALU
 547 #define ALU_HH ALU
 548
 549 #endif
 550
 551 #define ALU_I(thread, ip, operator)  \
 552 {                                                                              \
 553         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 554         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 555         uint64_t dst64 = *dst64_ptr;                                           \
 556         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 557         uint64_t dst = dst64 & dst64_mask;                                     \
 558                                                                                \
 559         uint64_t src = (ip)->alu.src_val;                                      \
 560                                                                                \
 561         uint64_t result = dst operator src;                                    \
 562                                                                                \
 563         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 564 }
 565
 566 #define ALU_MI ALU_I
 567
 568 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 569
 570 #define ALU_HI(thread, ip, operator)  \
 571 {                                                                              \
 572         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 573         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 574         uint64_t dst64 = *dst64_ptr;                                           \
 575         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 576         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 577                                                                                \
 578         uint64_t src = (ip)->alu.src_val;                                      \
 579                                                                                \
 580         uint64_t result = dst operator src;                                    \
 581         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 582                                                                                \
 583         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 584 }
 585
 586 #else
 587
 588 #define ALU_HI ALU_I
 589
 590 #endif
 591
 592 #define MOV(thread, ip)  \
 593 {                                                                              \
 594         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 595         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 596         uint64_t dst64 = *dst64_ptr;                                           \
 597         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 598                                                                                \
 599         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 600         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 601         uint64_t src64 = *src64_ptr;                                           \
 602         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
 603         uint64_t src = src64 & src64_mask;                                     \
 604                                                                                \
 605         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 606 }
 607
 608 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 609
 610 #define MOV_S(thread, ip)  \
 611 {                                                                              \
 612         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 613         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 614         uint64_t dst64 = *dst64_ptr;                                           \
 615         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 616                                                                                \
 617         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 618         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 619         uint64_t src64 = *src64_ptr;                                           \
 620         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
 621                                                                                \
 622         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 623 }
 624
 625 #else
 626
 627 #define MOV_S MOV
 628
 629 #endif
 630
 631 #define MOV_I(thread, ip)  \
 632 {                                                                              \
 633         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 634         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 635         uint64_t dst64 = *dst64_ptr;                                           \
 636         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 637                                                                                \
 638         uint64_t src = (ip)->mov.src_val;                                      \
 639                                                                                \
 640         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 641 }
 642
 643 #define METADATA_READ(thread, offset, n_bits)                                  \
 644 ({                                                                             \
 645         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 646         uint64_t m64 = *m64_ptr;                                               \
 647         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 648         (m64 & m64_mask);                                                      \
 649 })
 650
 651 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
 652 {                                                                              \
 653         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 654         uint64_t m64 = *m64_ptr;                                               \
 655         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 656                                                                                \
 657         uint64_t m_new = value;                                                \
 658                                                                                \
 659         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
 660 }
 661
 662 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 663 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 664 #endif
 665
 666 struct rte_swx_pipeline {
 667         struct struct_type_tailq struct_types;
 668         struct port_in_type_tailq port_in_types;
 669         struct port_in_tailq ports_in;
 670         struct port_out_type_tailq port_out_types;
 671         struct port_out_tailq ports_out;
 672         struct extern_type_tailq extern_types;
 673         struct extern_obj_tailq extern_objs;
 674         struct extern_func_tailq extern_funcs;
 675         struct header_tailq headers;
 676         struct struct_type *metadata_st;
 677         uint32_t metadata_struct_id;
 678         struct action_tailq actions;
 679         struct table_type_tailq table_types;
 680         struct table_tailq tables;
 681
 682         struct port_in_runtime *in;
 683         struct port_out_runtime *out;
 684         struct instruction **action_instructions;
 685         struct rte_swx_table_state *table_state;
 686         struct instruction *instructions;
 687         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 688
 689         uint32_t n_structs;
 690         uint32_t n_ports_in;
 691         uint32_t n_ports_out;
 692         uint32_t n_extern_objs;
 693         uint32_t n_extern_funcs;
 694         uint32_t n_actions;
 695         uint32_t n_tables;
 696         uint32_t n_headers;
 697         uint32_t thread_id;
 698         uint32_t port_id;
 699         uint32_t n_instructions;
 700         int build_done;
 701         int numa_node;
 702 };
 703
 704 /*
 705  * Struct.
 706  */
 707 static struct struct_type *
 708 struct_type_find(struct rte_swx_pipeline *p, const char *name)
 709 {
 710         struct struct_type *elem;
 711
 712         TAILQ_FOREACH(elem, &p->struct_types, node)
 713                 if (strcmp(elem->name, name) == 0)
 714                         return elem;
 715
 716         return NULL;
 717 }
 718
 719 static struct field *
 720 struct_type_field_find(struct struct_type *st, const char *name)
 721 {
 722         uint32_t i;
 723
 724         for (i = 0; i < st->n_fields; i++) {
 725                 struct field *f = &st->fields[i];
 726
 727                 if (strcmp(f->name, name) == 0)
 728                         return f;
 729         }
 730
 731         return NULL;
 732 }
 733
 734 int
 735 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 736                                       const char *name,
 737                                       struct rte_swx_field_params *fields,
 738                                       uint32_t n_fields)
 739 {
 740         struct struct_type *st;
 741         uint32_t i;
 742
 743         CHECK(p, EINVAL);
 744         CHECK_NAME(name, EINVAL);
 745         CHECK(fields, EINVAL);
 746         CHECK(n_fields, EINVAL);
 747
 748         for (i = 0; i < n_fields; i++) {
 749                 struct rte_swx_field_params *f = &fields[i];
 750                 uint32_t j;
 751
 752                 CHECK_NAME(f->name, EINVAL);
 753                 CHECK(f->n_bits, EINVAL);
 754                 CHECK(f->n_bits <= 64, EINVAL);
 755                 CHECK((f->n_bits & 7) == 0, EINVAL);
 756
 757                 for (j = 0; j < i; j++) {
 758                         struct rte_swx_field_params *f_prev = &fields[j];
 759
 760                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 761                 }
 762         }
 763
 764         CHECK(!struct_type_find(p, name), EEXIST);
 765
 766         /* Node allocation. */
 767         st = calloc(1, sizeof(struct struct_type));
 768         CHECK(st, ENOMEM);
 769
 770         st->fields = calloc(n_fields, sizeof(struct field));
 771         if (!st->fields) {
 772                 free(st);
 773                 CHECK(0, ENOMEM);
 774         }
 775
 776         /* Node initialization. */
 777         strcpy(st->name, name);
 778         for (i = 0; i < n_fields; i++) {
 779                 struct field *dst = &st->fields[i];
 780                 struct rte_swx_field_params *src = &fields[i];
 781
 782                 strcpy(dst->name, src->name);
 783                 dst->n_bits = src->n_bits;
 784                 dst->offset = st->n_bits;
 785
 786                 st->n_bits += src->n_bits;
 787         }
 788         st->n_fields = n_fields;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 792
 793         return 0;
 794 }
 795
 796 static int
 797 struct_build(struct rte_swx_pipeline *p)
 798 {
 799         uint32_t i;
 800
 801         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 802                 struct thread *t = &p->threads[i];
 803
 804                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 805                 CHECK(t->structs, ENOMEM);
 806         }
 807
 808         return 0;
 809 }
 810
 811 static void
 812 struct_build_free(struct rte_swx_pipeline *p)
 813 {
 814         uint32_t i;
 815
 816         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 817                 struct thread *t = &p->threads[i];
 818
 819                 free(t->structs);
 820                 t->structs = NULL;
 821         }
 822 }
 823
 824 static void
 825 struct_free(struct rte_swx_pipeline *p)
 826 {
 827         struct_build_free(p);
 828
 829         /* Struct types. */
 830         for ( ; ; ) {
 831                 struct struct_type *elem;
 832
 833                 elem = TAILQ_FIRST(&p->struct_types);
 834                 if (!elem)
 835                         break;
 836
 837                 TAILQ_REMOVE(&p->struct_types, elem, node);
 838                 free(elem->fields);
 839                 free(elem);
 840         }
 841 }
 842
 843 /*
 844  * Input port.
 845  */
 846 static struct port_in_type *
 847 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 848 {
 849         struct port_in_type *elem;
 850
 851         if (!name)
 852                 return NULL;
 853
 854         TAILQ_FOREACH(elem, &p->port_in_types, node)
 855                 if (strcmp(elem->name, name) == 0)
 856                         return elem;
 857
 858         return NULL;
 859 }
 860
 861 int
 862 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 863                                        const char *name,
 864                                        struct rte_swx_port_in_ops *ops)
 865 {
 866         struct port_in_type *elem;
 867
 868         CHECK(p, EINVAL);
 869         CHECK_NAME(name, EINVAL);
 870         CHECK(ops, EINVAL);
 871         CHECK(ops->create, EINVAL);
 872         CHECK(ops->free, EINVAL);
 873         CHECK(ops->pkt_rx, EINVAL);
 874         CHECK(ops->stats_read, EINVAL);
 875
 876         CHECK(!port_in_type_find(p, name), EEXIST);
 877
 878         /* Node allocation. */
 879         elem = calloc(1, sizeof(struct port_in_type));
 880         CHECK(elem, ENOMEM);
 881
 882         /* Node initialization. */
 883         strcpy(elem->name, name);
 884         memcpy(&elem->ops, ops, sizeof(*ops));
 885
 886         /* Node add to tailq. */
 887         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 888
 889         return 0;
 890 }
 891
 892 static struct port_in *
 893 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 894 {
 895         struct port_in *port;
 896
 897         TAILQ_FOREACH(port, &p->ports_in, node)
 898                 if (port->id == port_id)
 899                         return port;
 900
 901         return NULL;
 902 }
 903
 904 int
 905 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 906                                 uint32_t port_id,
 907                                 const char *port_type_name,
 908                                 void *args)
 909 {
 910         struct port_in_type *type = NULL;
 911         struct port_in *port = NULL;
 912         void *obj = NULL;
 913
 914         CHECK(p, EINVAL);
 915
 916         CHECK(!port_in_find(p, port_id), EINVAL);
 917
 918         CHECK_NAME(port_type_name, EINVAL);
 919         type = port_in_type_find(p, port_type_name);
 920         CHECK(type, EINVAL);
 921
 922         obj = type->ops.create(args);
 923         CHECK(obj, ENODEV);
 924
 925         /* Node allocation. */
 926         port = calloc(1, sizeof(struct port_in));
 927         CHECK(port, ENOMEM);
 928
 929         /* Node initialization. */
 930         port->type = type;
 931         port->obj = obj;
 932         port->id = port_id;
 933
 934         /* Node add to tailq. */
 935         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 936         if (p->n_ports_in < port_id + 1)
 937                 p->n_ports_in = port_id + 1;
 938
 939         return 0;
 940 }
 941
 942 static int
 943 port_in_build(struct rte_swx_pipeline *p)
 944 {
 945         struct port_in *port;
 946         uint32_t i;
 947
 948         CHECK(p->n_ports_in, EINVAL);
 949         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 950
 951         for (i = 0; i < p->n_ports_in; i++)
 952                 CHECK(port_in_find(p, i), EINVAL);
 953
 954         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 955         CHECK(p->in, ENOMEM);
 956
 957         TAILQ_FOREACH(port, &p->ports_in, node) {
 958                 struct port_in_runtime *in = &p->in[port->id];
 959
 960                 in->pkt_rx = port->type->ops.pkt_rx;
 961                 in->obj = port->obj;
 962         }
 963
 964         return 0;
 965 }
 966
 967 static void
 968 port_in_build_free(struct rte_swx_pipeline *p)
 969 {
 970         free(p->in);
 971         p->in = NULL;
 972 }
 973
 974 static void
 975 port_in_free(struct rte_swx_pipeline *p)
 976 {
 977         port_in_build_free(p);
 978
 979         /* Input ports. */
 980         for ( ; ; ) {
 981                 struct port_in *port;
 982
 983                 port = TAILQ_FIRST(&p->ports_in);
 984                 if (!port)
 985                         break;
 986
 987                 TAILQ_REMOVE(&p->ports_in, port, node);
 988                 port->type->ops.free(port->obj);
 989                 free(port);
 990         }
 991
 992         /* Input port types. */
 993         for ( ; ; ) {
 994                 struct port_in_type *elem;
 995
 996                 elem = TAILQ_FIRST(&p->port_in_types);
 997                 if (!elem)
 998                         break;
 999
1000                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1001                 free(elem);
1002         }
1003 }
1004
1005 /*
1006  * Output port.
1007  */
1008 static struct port_out_type *
1009 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1010 {
1011         struct port_out_type *elem;
1012
1013         if (!name)
1014                 return NULL;
1015
1016         TAILQ_FOREACH(elem, &p->port_out_types, node)
1017                 if (!strcmp(elem->name, name))
1018                         return elem;
1019
1020         return NULL;
1021 }
1022
1023 int
1024 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1025                                         const char *name,
1026                                         struct rte_swx_port_out_ops *ops)
1027 {
1028         struct port_out_type *elem;
1029
1030         CHECK(p, EINVAL);
1031         CHECK_NAME(name, EINVAL);
1032         CHECK(ops, EINVAL);
1033         CHECK(ops->create, EINVAL);
1034         CHECK(ops->free, EINVAL);
1035         CHECK(ops->pkt_tx, EINVAL);
1036         CHECK(ops->stats_read, EINVAL);
1037
1038         CHECK(!port_out_type_find(p, name), EEXIST);
1039
1040         /* Node allocation. */
1041         elem = calloc(1, sizeof(struct port_out_type));
1042         CHECK(elem, ENOMEM);
1043
1044         /* Node initialization. */
1045         strcpy(elem->name, name);
1046         memcpy(&elem->ops, ops, sizeof(*ops));
1047
1048         /* Node add to tailq. */
1049         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1050
1051         return 0;
1052 }
1053
1054 static struct port_out *
1055 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1056 {
1057         struct port_out *port;
1058
1059         TAILQ_FOREACH(port, &p->ports_out, node)
1060                 if (port->id == port_id)
1061                         return port;
1062
1063         return NULL;
1064 }
1065
1066 int
1067 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1068                                  uint32_t port_id,
1069                                  const char *port_type_name,
1070                                  void *args)
1071 {
1072         struct port_out_type *type = NULL;
1073         struct port_out *port = NULL;
1074         void *obj = NULL;
1075
1076         CHECK(p, EINVAL);
1077
1078         CHECK(!port_out_find(p, port_id), EINVAL);
1079
1080         CHECK_NAME(port_type_name, EINVAL);
1081         type = port_out_type_find(p, port_type_name);
1082         CHECK(type, EINVAL);
1083
1084         obj = type->ops.create(args);
1085         CHECK(obj, ENODEV);
1086
1087         /* Node allocation. */
1088         port = calloc(1, sizeof(struct port_out));
1089         CHECK(port, ENOMEM);
1090
1091         /* Node initialization. */
1092         port->type = type;
1093         port->obj = obj;
1094         port->id = port_id;
1095
1096         /* Node add to tailq. */
1097         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1098         if (p->n_ports_out < port_id + 1)
1099                 p->n_ports_out = port_id + 1;
1100
1101         return 0;
1102 }
1103
1104 static int
1105 port_out_build(struct rte_swx_pipeline *p)
1106 {
1107         struct port_out *port;
1108         uint32_t i;
1109
1110         CHECK(p->n_ports_out, EINVAL);
1111
1112         for (i = 0; i < p->n_ports_out; i++)
1113                 CHECK(port_out_find(p, i), EINVAL);
1114
1115         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1116         CHECK(p->out, ENOMEM);
1117
1118         TAILQ_FOREACH(port, &p->ports_out, node) {
1119                 struct port_out_runtime *out = &p->out[port->id];
1120
1121                 out->pkt_tx = port->type->ops.pkt_tx;
1122                 out->flush = port->type->ops.flush;
1123                 out->obj = port->obj;
1124         }
1125
1126         return 0;
1127 }
1128
1129 static void
1130 port_out_build_free(struct rte_swx_pipeline *p)
1131 {
1132         free(p->out);
1133         p->out = NULL;
1134 }
1135
1136 static void
1137 port_out_free(struct rte_swx_pipeline *p)
1138 {
1139         port_out_build_free(p);
1140
1141         /* Output ports. */
1142         for ( ; ; ) {
1143                 struct port_out *port;
1144
1145                 port = TAILQ_FIRST(&p->ports_out);
1146                 if (!port)
1147                         break;
1148
1149                 TAILQ_REMOVE(&p->ports_out, port, node);
1150                 port->type->ops.free(port->obj);
1151                 free(port);
1152         }
1153
1154         /* Output port types. */
1155         for ( ; ; ) {
1156                 struct port_out_type *elem;
1157
1158                 elem = TAILQ_FIRST(&p->port_out_types);
1159                 if (!elem)
1160                         break;
1161
1162                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1163                 free(elem);
1164         }
1165 }
1166
1167 /*
1168  * Extern object.
1169  */
1170 static struct extern_type *
1171 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1172 {
1173         struct extern_type *elem;
1174
1175         TAILQ_FOREACH(elem, &p->extern_types, node)
1176                 if (strcmp(elem->name, name) == 0)
1177                         return elem;
1178
1179         return NULL;
1180 }
1181
1182 static struct extern_type_member_func *
1183 extern_type_member_func_find(struct extern_type *type, const char *name)
1184 {
1185         struct extern_type_member_func *elem;
1186
1187         TAILQ_FOREACH(elem, &type->funcs, node)
1188                 if (strcmp(elem->name, name) == 0)
1189                         return elem;
1190
1191         return NULL;
1192 }
1193
1194 static struct extern_obj *
1195 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1196 {
1197         struct extern_obj *elem;
1198
1199         TAILQ_FOREACH(elem, &p->extern_objs, node)
1200                 if (strcmp(elem->name, name) == 0)
1201                         return elem;
1202
1203         return NULL;
1204 }
1205
1206 static struct field *
1207 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1208                                const char *name,
1209                                struct extern_obj **object)
1210 {
1211         struct extern_obj *obj;
1212         struct field *f;
1213         char *obj_name, *field_name;
1214
1215         if ((name[0] != 'e') || (name[1] != '.'))
1216                 return NULL;
1217
1218         obj_name = strdup(&name[2]);
1219         if (!obj_name)
1220                 return NULL;
1221
1222         field_name = strchr(obj_name, '.');
1223         if (!field_name) {
1224                 free(obj_name);
1225                 return NULL;
1226         }
1227
1228         *field_name = 0;
1229         field_name++;
1230
1231         obj = extern_obj_find(p, obj_name);
1232         if (!obj) {
1233                 free(obj_name);
1234                 return NULL;
1235         }
1236
1237         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1238         if (!f) {
1239                 free(obj_name);
1240                 return NULL;
1241         }
1242
1243         if (object)
1244                 *object = obj;
1245
1246         free(obj_name);
1247         return f;
1248 }
1249
1250 int
1251 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1252         const char *name,
1253         const char *mailbox_struct_type_name,
1254         rte_swx_extern_type_constructor_t constructor,
1255         rte_swx_extern_type_destructor_t destructor)
1256 {
1257         struct extern_type *elem;
1258         struct struct_type *mailbox_struct_type;
1259
1260         CHECK(p, EINVAL);
1261
1262         CHECK_NAME(name, EINVAL);
1263         CHECK(!extern_type_find(p, name), EEXIST);
1264
1265         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1266         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1267         CHECK(mailbox_struct_type, EINVAL);
1268
1269         CHECK(constructor, EINVAL);
1270         CHECK(destructor, EINVAL);
1271
1272         /* Node allocation. */
1273         elem = calloc(1, sizeof(struct extern_type));
1274         CHECK(elem, ENOMEM);
1275
1276         /* Node initialization. */
1277         strcpy(elem->name, name);
1278         elem->mailbox_struct_type = mailbox_struct_type;
1279         elem->constructor = constructor;
1280         elem->destructor = destructor;
1281         TAILQ_INIT(&elem->funcs);
1282
1283         /* Node add to tailq. */
1284         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1285
1286         return 0;
1287 }
1288
1289 int
1290 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1291         const char *extern_type_name,
1292         const char *name,
1293         rte_swx_extern_type_member_func_t member_func)
1294 {
1295         struct extern_type *type;
1296         struct extern_type_member_func *type_member;
1297
1298         CHECK(p, EINVAL);
1299
1300         CHECK(extern_type_name, EINVAL);
1301         type = extern_type_find(p, extern_type_name);
1302         CHECK(type, EINVAL);
1303         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1304
1305         CHECK(name, EINVAL);
1306         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1307
1308         CHECK(member_func, EINVAL);
1309
1310         /* Node allocation. */
1311         type_member = calloc(1, sizeof(struct extern_type_member_func));
1312         CHECK(type_member, ENOMEM);
1313
1314         /* Node initialization. */
1315         strcpy(type_member->name, name);
1316         type_member->func = member_func;
1317         type_member->id = type->n_funcs;
1318
1319         /* Node add to tailq. */
1320         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1321         type->n_funcs++;
1322
1323         return 0;
1324 }
1325
1326 int
1327 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1328                                       const char *extern_type_name,
1329                                       const char *name,
1330                                       const char *args)
1331 {
1332         struct extern_type *type;
1333         struct extern_obj *obj;
1334         void *obj_handle;
1335
1336         CHECK(p, EINVAL);
1337
1338         CHECK_NAME(extern_type_name, EINVAL);
1339         type = extern_type_find(p, extern_type_name);
1340         CHECK(type, EINVAL);
1341
1342         CHECK_NAME(name, EINVAL);
1343         CHECK(!extern_obj_find(p, name), EEXIST);
1344
1345         /* Node allocation. */
1346         obj = calloc(1, sizeof(struct extern_obj));
1347         CHECK(obj, ENOMEM);
1348
1349         /* Object construction. */
1350         obj_handle = type->constructor(args);
1351         if (!obj_handle) {
1352                 free(obj);
1353                 CHECK(0, ENODEV);
1354         }
1355
1356         /* Node initialization. */
1357         strcpy(obj->name, name);
1358         obj->type = type;
1359         obj->obj = obj_handle;
1360         obj->struct_id = p->n_structs;
1361         obj->id = p->n_extern_objs;
1362
1363         /* Node add to tailq. */
1364         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1365         p->n_extern_objs++;
1366         p->n_structs++;
1367
1368         return 0;
1369 }
1370
1371 static int
1372 extern_obj_build(struct rte_swx_pipeline *p)
1373 {
1374         uint32_t i;
1375
1376         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1377                 struct thread *t = &p->threads[i];
1378                 struct extern_obj *obj;
1379
1380                 t->extern_objs = calloc(p->n_extern_objs,
1381                                         sizeof(struct extern_obj_runtime));
1382                 CHECK(t->extern_objs, ENOMEM);
1383
1384                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1385                         struct extern_obj_runtime *r =
1386                                 &t->extern_objs[obj->id];
1387                         struct extern_type_member_func *func;
1388                         uint32_t mailbox_size =
1389                                 obj->type->mailbox_struct_type->n_bits / 8;
1390
1391                         r->obj = obj->obj;
1392
1393                         r->mailbox = calloc(1, mailbox_size);
1394                         CHECK(r->mailbox, ENOMEM);
1395
1396                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1397                                 r->funcs[func->id] = func->func;
1398
1399                         t->structs[obj->struct_id] = r->mailbox;
1400                 }
1401         }
1402
1403         return 0;
1404 }
1405
1406 static void
1407 extern_obj_build_free(struct rte_swx_pipeline *p)
1408 {
1409         uint32_t i;
1410
1411         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1412                 struct thread *t = &p->threads[i];
1413                 uint32_t j;
1414
1415                 if (!t->extern_objs)
1416                         continue;
1417
1418                 for (j = 0; j < p->n_extern_objs; j++) {
1419                         struct extern_obj_runtime *r = &t->extern_objs[j];
1420
1421                         free(r->mailbox);
1422                 }
1423
1424                 free(t->extern_objs);
1425                 t->extern_objs = NULL;
1426         }
1427 }
1428
1429 static void
1430 extern_obj_free(struct rte_swx_pipeline *p)
1431 {
1432         extern_obj_build_free(p);
1433
1434         /* Extern objects. */
1435         for ( ; ; ) {
1436                 struct extern_obj *elem;
1437
1438                 elem = TAILQ_FIRST(&p->extern_objs);
1439                 if (!elem)
1440                         break;
1441
1442                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1443                 if (elem->obj)
1444                         elem->type->destructor(elem->obj);
1445                 free(elem);
1446         }
1447
1448         /* Extern types. */
1449         for ( ; ; ) {
1450                 struct extern_type *elem;
1451
1452                 elem = TAILQ_FIRST(&p->extern_types);
1453                 if (!elem)
1454                         break;
1455
1456                 TAILQ_REMOVE(&p->extern_types, elem, node);
1457
1458                 for ( ; ; ) {
1459                         struct extern_type_member_func *func;
1460
1461                         func = TAILQ_FIRST(&elem->funcs);
1462                         if (!func)
1463                                 break;
1464
1465                         TAILQ_REMOVE(&elem->funcs, func, node);
1466                         free(func);
1467                 }
1468
1469                 free(elem);
1470         }
1471 }
1472
1473 /*
1474  * Extern function.
1475  */
1476 static struct extern_func *
1477 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1478 {
1479         struct extern_func *elem;
1480
1481         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1482                 if (strcmp(elem->name, name) == 0)
1483                         return elem;
1484
1485         return NULL;
1486 }
1487
1488 static struct field *
1489 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1490                                 const char *name,
1491                                 struct extern_func **function)
1492 {
1493         struct extern_func *func;
1494         struct field *f;
1495         char *func_name, *field_name;
1496
1497         if ((name[0] != 'f') || (name[1] != '.'))
1498                 return NULL;
1499
1500         func_name = strdup(&name[2]);
1501         if (!func_name)
1502                 return NULL;
1503
1504         field_name = strchr(func_name, '.');
1505         if (!field_name) {
1506                 free(func_name);
1507                 return NULL;
1508         }
1509
1510         *field_name = 0;
1511         field_name++;
1512
1513         func = extern_func_find(p, func_name);
1514         if (!func) {
1515                 free(func_name);
1516                 return NULL;
1517         }
1518
1519         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1520         if (!f) {
1521                 free(func_name);
1522                 return NULL;
1523         }
1524
1525         if (function)
1526                 *function = func;
1527
1528         free(func_name);
1529         return f;
1530 }
1531
1532 int
1533 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1534                                       const char *name,
1535                                       const char *mailbox_struct_type_name,
1536                                       rte_swx_extern_func_t func)
1537 {
1538         struct extern_func *f;
1539         struct struct_type *mailbox_struct_type;
1540
1541         CHECK(p, EINVAL);
1542
1543         CHECK_NAME(name, EINVAL);
1544         CHECK(!extern_func_find(p, name), EEXIST);
1545
1546         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1547         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1548         CHECK(mailbox_struct_type, EINVAL);
1549
1550         CHECK(func, EINVAL);
1551
1552         /* Node allocation. */
1553         f = calloc(1, sizeof(struct extern_func));
1554         CHECK(func, ENOMEM);
1555
1556         /* Node initialization. */
1557         strcpy(f->name, name);
1558         f->mailbox_struct_type = mailbox_struct_type;
1559         f->func = func;
1560         f->struct_id = p->n_structs;
1561         f->id = p->n_extern_funcs;
1562
1563         /* Node add to tailq. */
1564         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1565         p->n_extern_funcs++;
1566         p->n_structs++;
1567
1568         return 0;
1569 }
1570
1571 static int
1572 extern_func_build(struct rte_swx_pipeline *p)
1573 {
1574         uint32_t i;
1575
1576         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1577                 struct thread *t = &p->threads[i];
1578                 struct extern_func *func;
1579
1580                 /* Memory allocation. */
1581                 t->extern_funcs = calloc(p->n_extern_funcs,
1582                                          sizeof(struct extern_func_runtime));
1583                 CHECK(t->extern_funcs, ENOMEM);
1584
1585                 /* Extern function. */
1586                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1587                         struct extern_func_runtime *r =
1588                                 &t->extern_funcs[func->id];
1589                         uint32_t mailbox_size =
1590                                 func->mailbox_struct_type->n_bits / 8;
1591
1592                         r->func = func->func;
1593
1594                         r->mailbox = calloc(1, mailbox_size);
1595                         CHECK(r->mailbox, ENOMEM);
1596
1597                         t->structs[func->struct_id] = r->mailbox;
1598                 }
1599         }
1600
1601         return 0;
1602 }
1603
1604 static void
1605 extern_func_build_free(struct rte_swx_pipeline *p)
1606 {
1607         uint32_t i;
1608
1609         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1610                 struct thread *t = &p->threads[i];
1611                 uint32_t j;
1612
1613                 if (!t->extern_funcs)
1614                         continue;
1615
1616                 for (j = 0; j < p->n_extern_funcs; j++) {
1617                         struct extern_func_runtime *r = &t->extern_funcs[j];
1618
1619                         free(r->mailbox);
1620                 }
1621
1622                 free(t->extern_funcs);
1623                 t->extern_funcs = NULL;
1624         }
1625 }
1626
1627 static void
1628 extern_func_free(struct rte_swx_pipeline *p)
1629 {
1630         extern_func_build_free(p);
1631
1632         for ( ; ; ) {
1633                 struct extern_func *elem;
1634
1635                 elem = TAILQ_FIRST(&p->extern_funcs);
1636                 if (!elem)
1637                         break;
1638
1639                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1640                 free(elem);
1641         }
1642 }
1643
1644 /*
1645  * Header.
1646  */
1647 static struct header *
1648 header_find(struct rte_swx_pipeline *p, const char *name)
1649 {
1650         struct header *elem;
1651
1652         TAILQ_FOREACH(elem, &p->headers, node)
1653                 if (strcmp(elem->name, name) == 0)
1654                         return elem;
1655
1656         return NULL;
1657 }
1658
1659 static struct header *
1660 header_parse(struct rte_swx_pipeline *p,
1661              const char *name)
1662 {
1663         if (name[0] != 'h' || name[1] != '.')
1664                 return NULL;
1665
1666         return header_find(p, &name[2]);
1667 }
1668
1669 static struct field *
1670 header_field_parse(struct rte_swx_pipeline *p,
1671                    const char *name,
1672                    struct header **header)
1673 {
1674         struct header *h;
1675         struct field *f;
1676         char *header_name, *field_name;
1677
1678         if ((name[0] != 'h') || (name[1] != '.'))
1679                 return NULL;
1680
1681         header_name = strdup(&name[2]);
1682         if (!header_name)
1683                 return NULL;
1684
1685         field_name = strchr(header_name, '.');
1686         if (!field_name) {
1687                 free(header_name);
1688                 return NULL;
1689         }
1690
1691         *field_name = 0;
1692         field_name++;
1693
1694         h = header_find(p, header_name);
1695         if (!h) {
1696                 free(header_name);
1697                 return NULL;
1698         }
1699
1700         f = struct_type_field_find(h->st, field_name);
1701         if (!f) {
1702                 free(header_name);
1703                 return NULL;
1704         }
1705
1706         if (header)
1707                 *header = h;
1708
1709         free(header_name);
1710         return f;
1711 }
1712
1713 int
1714 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1715                                         const char *name,
1716                                         const char *struct_type_name)
1717 {
1718         struct struct_type *st;
1719         struct header *h;
1720         size_t n_headers_max;
1721
1722         CHECK(p, EINVAL);
1723         CHECK_NAME(name, EINVAL);
1724         CHECK_NAME(struct_type_name, EINVAL);
1725
1726         CHECK(!header_find(p, name), EEXIST);
1727
1728         st = struct_type_find(p, struct_type_name);
1729         CHECK(st, EINVAL);
1730
1731         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1732         CHECK(p->n_headers < n_headers_max, ENOSPC);
1733
1734         /* Node allocation. */
1735         h = calloc(1, sizeof(struct header));
1736         CHECK(h, ENOMEM);
1737
1738         /* Node initialization. */
1739         strcpy(h->name, name);
1740         h->st = st;
1741         h->struct_id = p->n_structs;
1742         h->id = p->n_headers;
1743
1744         /* Node add to tailq. */
1745         TAILQ_INSERT_TAIL(&p->headers, h, node);
1746         p->n_headers++;
1747         p->n_structs++;
1748
1749         return 0;
1750 }
1751
1752 static int
1753 header_build(struct rte_swx_pipeline *p)
1754 {
1755         struct header *h;
1756         uint32_t n_bytes = 0, i;
1757
1758         TAILQ_FOREACH(h, &p->headers, node) {
1759                 n_bytes += h->st->n_bits / 8;
1760         }
1761
1762         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1763                 struct thread *t = &p->threads[i];
1764                 uint32_t offset = 0;
1765
1766                 t->headers = calloc(p->n_headers,
1767                                     sizeof(struct header_runtime));
1768                 CHECK(t->headers, ENOMEM);
1769
1770                 t->headers_out = calloc(p->n_headers,
1771                                         sizeof(struct header_out_runtime));
1772                 CHECK(t->headers_out, ENOMEM);
1773
1774                 t->header_storage = calloc(1, n_bytes);
1775                 CHECK(t->header_storage, ENOMEM);
1776
1777                 t->header_out_storage = calloc(1, n_bytes);
1778                 CHECK(t->header_out_storage, ENOMEM);
1779
1780                 TAILQ_FOREACH(h, &p->headers, node) {
1781                         uint8_t *header_storage;
1782
1783                         header_storage = &t->header_storage[offset];
1784                         offset += h->st->n_bits / 8;
1785
1786                         t->headers[h->id].ptr0 = header_storage;
1787                         t->structs[h->struct_id] = header_storage;
1788                 }
1789         }
1790
1791         return 0;
1792 }
1793
1794 static void
1795 header_build_free(struct rte_swx_pipeline *p)
1796 {
1797         uint32_t i;
1798
1799         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1800                 struct thread *t = &p->threads[i];
1801
1802                 free(t->headers_out);
1803                 t->headers_out = NULL;
1804
1805                 free(t->headers);
1806                 t->headers = NULL;
1807
1808                 free(t->header_out_storage);
1809                 t->header_out_storage = NULL;
1810
1811                 free(t->header_storage);
1812                 t->header_storage = NULL;
1813         }
1814 }
1815
1816 static void
1817 header_free(struct rte_swx_pipeline *p)
1818 {
1819         header_build_free(p);
1820
1821         for ( ; ; ) {
1822                 struct header *elem;
1823
1824                 elem = TAILQ_FIRST(&p->headers);
1825                 if (!elem)
1826                         break;
1827
1828                 TAILQ_REMOVE(&p->headers, elem, node);
1829                 free(elem);
1830         }
1831 }
1832
1833 /*
1834  * Meta-data.
1835  */
1836 static struct field *
1837 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1838 {
1839         if (!p->metadata_st)
1840                 return NULL;
1841
1842         if (name[0] != 'm' || name[1] != '.')
1843                 return NULL;
1844
1845         return struct_type_field_find(p->metadata_st, &name[2]);
1846 }
1847
1848 int
1849 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1850                                           const char *struct_type_name)
1851 {
1852         struct struct_type *st = NULL;
1853
1854         CHECK(p, EINVAL);
1855
1856         CHECK_NAME(struct_type_name, EINVAL);
1857         st  = struct_type_find(p, struct_type_name);
1858         CHECK(st, EINVAL);
1859         CHECK(!p->metadata_st, EINVAL);
1860
1861         p->metadata_st = st;
1862         p->metadata_struct_id = p->n_structs;
1863
1864         p->n_structs++;
1865
1866         return 0;
1867 }
1868
1869 static int
1870 metadata_build(struct rte_swx_pipeline *p)
1871 {
1872         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1873         uint32_t i;
1874
1875         /* Thread-level initialization. */
1876         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1877                 struct thread *t = &p->threads[i];
1878                 uint8_t *metadata;
1879
1880                 metadata = calloc(1, n_bytes);
1881                 CHECK(metadata, ENOMEM);
1882
1883                 t->metadata = metadata;
1884                 t->structs[p->metadata_struct_id] = metadata;
1885         }
1886
1887         return 0;
1888 }
1889
1890 static void
1891 metadata_build_free(struct rte_swx_pipeline *p)
1892 {
1893         uint32_t i;
1894
1895         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1896                 struct thread *t = &p->threads[i];
1897
1898                 free(t->metadata);
1899                 t->metadata = NULL;
1900         }
1901 }
1902
1903 static void
1904 metadata_free(struct rte_swx_pipeline *p)
1905 {
1906         metadata_build_free(p);
1907 }
1908
1909 /*
1910  * Instruction.
1911  */
1912 static struct field *
1913 action_field_parse(struct action *action, const char *name);
1914
1915 static struct field *
1916 struct_field_parse(struct rte_swx_pipeline *p,
1917                    struct action *action,
1918                    const char *name,
1919                    uint32_t *struct_id)
1920 {
1921         struct field *f;
1922
1923         switch (name[0]) {
1924         case 'h':
1925         {
1926                 struct header *header;
1927
1928                 f = header_field_parse(p, name, &header);
1929                 if (!f)
1930                         return NULL;
1931
1932                 *struct_id = header->struct_id;
1933                 return f;
1934         }
1935
1936         case 'm':
1937         {
1938                 f = metadata_field_parse(p, name);
1939                 if (!f)
1940                         return NULL;
1941
1942                 *struct_id = p->metadata_struct_id;
1943                 return f;
1944         }
1945
1946         case 't':
1947         {
1948                 if (!action)
1949                         return NULL;
1950
1951                 f = action_field_parse(action, name);
1952                 if (!f)
1953                         return NULL;
1954
1955                 *struct_id = 0;
1956                 return f;
1957         }
1958
1959         case 'e':
1960         {
1961                 struct extern_obj *obj;
1962
1963                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1964                 if (!f)
1965                         return NULL;
1966
1967                 *struct_id = obj->struct_id;
1968                 return f;
1969         }
1970
1971         case 'f':
1972         {
1973                 struct extern_func *func;
1974
1975                 f = extern_func_mailbox_field_parse(p, name, &func);
1976                 if (!f)
1977                         return NULL;
1978
1979                 *struct_id = func->struct_id;
1980                 return f;
1981         }
1982
1983         default:
1984                 return NULL;
1985         }
1986 }
1987
1988 static inline void
1989 pipeline_port_inc(struct rte_swx_pipeline *p)
1990 {
1991         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
1992 }
1993
1994 static inline void
1995 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1996 {
1997         t->ip = p->instructions;
1998 }
1999
2000 static inline void
2001 thread_ip_inc(struct rte_swx_pipeline *p);
2002
2003 static inline void
2004 thread_ip_inc(struct rte_swx_pipeline *p)
2005 {
2006         struct thread *t = &p->threads[p->thread_id];
2007
2008         t->ip++;
2009 }
2010
2011 static inline void
2012 thread_ip_inc_cond(struct thread *t, int cond)
2013 {
2014         t->ip += cond;
2015 }
2016
2017 static inline void
2018 thread_yield(struct rte_swx_pipeline *p)
2019 {
2020         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2021 }
2022
2023 /*
2024  * rx.
2025  */
2026 static int
2027 instr_rx_translate(struct rte_swx_pipeline *p,
2028                    struct action *action,
2029                    char **tokens,
2030                    int n_tokens,
2031                    struct instruction *instr,
2032                    struct instruction_data *data __rte_unused)
2033 {
2034         struct field *f;
2035
2036         CHECK(!action, EINVAL);
2037         CHECK(n_tokens == 2, EINVAL);
2038
2039         f = metadata_field_parse(p, tokens[1]);
2040         CHECK(f, EINVAL);
2041
2042         instr->type = INSTR_RX;
2043         instr->io.io.offset = f->offset / 8;
2044         instr->io.io.n_bits = f->n_bits;
2045         return 0;
2046 }
2047
2048 static inline void
2049 instr_rx_exec(struct rte_swx_pipeline *p);
2050
2051 static inline void
2052 instr_rx_exec(struct rte_swx_pipeline *p)
2053 {
2054         struct thread *t = &p->threads[p->thread_id];
2055         struct instruction *ip = t->ip;
2056         struct port_in_runtime *port = &p->in[p->port_id];
2057         struct rte_swx_pkt *pkt = &t->pkt;
2058         int pkt_received;
2059
2060         /* Packet. */
2061         pkt_received = port->pkt_rx(port->obj, pkt);
2062         t->ptr = &pkt->pkt[pkt->offset];
2063         rte_prefetch0(t->ptr);
2064
2065         TRACE("[Thread %2u] rx %s from port %u\n",
2066               p->thread_id,
2067               pkt_received ? "1 pkt" : "0 pkts",
2068               p->port_id);
2069
2070         /* Headers. */
2071         t->valid_headers = 0;
2072         t->n_headers_out = 0;
2073
2074         /* Meta-data. */
2075         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2076
2077         /* Tables. */
2078         t->table_state = p->table_state;
2079
2080         /* Thread. */
2081         pipeline_port_inc(p);
2082         thread_ip_inc_cond(t, pkt_received);
2083         thread_yield(p);
2084 }
2085
2086 /*
2087  * tx.
2088  */
2089 static int
2090 instr_tx_translate(struct rte_swx_pipeline *p,
2091                    struct action *action __rte_unused,
2092                    char **tokens,
2093                    int n_tokens,
2094                    struct instruction *instr,
2095                    struct instruction_data *data __rte_unused)
2096 {
2097         struct field *f;
2098
2099         CHECK(n_tokens == 2, EINVAL);
2100
2101         f = metadata_field_parse(p, tokens[1]);
2102         CHECK(f, EINVAL);
2103
2104         instr->type = INSTR_TX;
2105         instr->io.io.offset = f->offset / 8;
2106         instr->io.io.n_bits = f->n_bits;
2107         return 0;
2108 }
2109
2110 static inline void
2111 emit_handler(struct thread *t)
2112 {
2113         struct header_out_runtime *h0 = &t->headers_out[0];
2114         struct header_out_runtime *h1 = &t->headers_out[1];
2115         uint32_t offset = 0, i;
2116
2117         /* No header change or header decapsulation. */
2118         if ((t->n_headers_out == 1) &&
2119             (h0->ptr + h0->n_bytes == t->ptr)) {
2120                 TRACE("Emit handler: no header change or header decap.\n");
2121
2122                 t->pkt.offset -= h0->n_bytes;
2123                 t->pkt.length += h0->n_bytes;
2124
2125                 return;
2126         }
2127
2128         /* Header encapsulation (optionally, with prior header decasulation). */
2129         if ((t->n_headers_out == 2) &&
2130             (h1->ptr + h1->n_bytes == t->ptr) &&
2131             (h0->ptr == h0->ptr0)) {
2132                 uint32_t offset;
2133
2134                 TRACE("Emit handler: header encapsulation.\n");
2135
2136                 offset = h0->n_bytes + h1->n_bytes;
2137                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2138                 t->pkt.offset -= offset;
2139                 t->pkt.length += offset;
2140
2141                 return;
2142         }
2143
2144         /* Header insertion. */
2145         /* TBD */
2146
2147         /* Header extraction. */
2148         /* TBD */
2149
2150         /* For any other case. */
2151         TRACE("Emit handler: complex case.\n");
2152
2153         for (i = 0; i < t->n_headers_out; i++) {
2154                 struct header_out_runtime *h = &t->headers_out[i];
2155
2156                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2157                 offset += h->n_bytes;
2158         }
2159
2160         if (offset) {
2161                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2162                 t->pkt.offset -= offset;
2163                 t->pkt.length += offset;
2164         }
2165 }
2166
2167 static inline void
2168 instr_tx_exec(struct rte_swx_pipeline *p);
2169
2170 static inline void
2171 instr_tx_exec(struct rte_swx_pipeline *p)
2172 {
2173         struct thread *t = &p->threads[p->thread_id];
2174         struct instruction *ip = t->ip;
2175         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2176         struct port_out_runtime *port = &p->out[port_id];
2177         struct rte_swx_pkt *pkt = &t->pkt;
2178
2179         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2180               p->thread_id,
2181               (uint32_t)port_id);
2182
2183         /* Headers. */
2184         emit_handler(t);
2185
2186         /* Packet. */
2187         port->pkt_tx(port->obj, pkt);
2188
2189         /* Thread. */
2190         thread_ip_reset(p, t);
2191         instr_rx_exec(p);
2192 }
2193
2194 /*
2195  * extract.
2196  */
2197 static int
2198 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2199                             struct action *action,
2200                             char **tokens,
2201                             int n_tokens,
2202                             struct instruction *instr,
2203                             struct instruction_data *data __rte_unused)
2204 {
2205         struct header *h;
2206
2207         CHECK(!action, EINVAL);
2208         CHECK(n_tokens == 2, EINVAL);
2209
2210         h = header_parse(p, tokens[1]);
2211         CHECK(h, EINVAL);
2212
2213         instr->type = INSTR_HDR_EXTRACT;
2214         instr->io.hdr.header_id[0] = h->id;
2215         instr->io.hdr.struct_id[0] = h->struct_id;
2216         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2217         return 0;
2218 }
2219
2220 static inline void
2221 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2222
2223 static inline void
2224 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2225 {
2226         struct thread *t = &p->threads[p->thread_id];
2227         struct instruction *ip = t->ip;
2228         uint64_t valid_headers = t->valid_headers;
2229         uint8_t *ptr = t->ptr;
2230         uint32_t offset = t->pkt.offset;
2231         uint32_t length = t->pkt.length;
2232         uint32_t i;
2233
2234         for (i = 0; i < n_extract; i++) {
2235                 uint32_t header_id = ip->io.hdr.header_id[i];
2236                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2237                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2238
2239                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2240                       p->thread_id,
2241                       header_id,
2242                       n_bytes);
2243
2244                 /* Headers. */
2245                 t->structs[struct_id] = ptr;
2246                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2247
2248                 /* Packet. */
2249                 offset += n_bytes;
2250                 length -= n_bytes;
2251                 ptr += n_bytes;
2252         }
2253
2254         /* Headers. */
2255         t->valid_headers = valid_headers;
2256
2257         /* Packet. */
2258         t->pkt.offset = offset;
2259         t->pkt.length = length;
2260         t->ptr = ptr;
2261 }
2262
2263 static inline void
2264 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2265 {
2266         __instr_hdr_extract_exec(p, 1);
2267
2268         /* Thread. */
2269         thread_ip_inc(p);
2270 }
2271
2272 static inline void
2273 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2274 {
2275         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2276               p->thread_id);
2277
2278         __instr_hdr_extract_exec(p, 2);
2279
2280         /* Thread. */
2281         thread_ip_inc(p);
2282 }
2283
2284 static inline void
2285 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2286 {
2287         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2288               p->thread_id);
2289
2290         __instr_hdr_extract_exec(p, 3);
2291
2292         /* Thread. */
2293         thread_ip_inc(p);
2294 }
2295
2296 static inline void
2297 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2298 {
2299         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2300               p->thread_id);
2301
2302         __instr_hdr_extract_exec(p, 4);
2303
2304         /* Thread. */
2305         thread_ip_inc(p);
2306 }
2307
2308 static inline void
2309 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2310 {
2311         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2312               p->thread_id);
2313
2314         __instr_hdr_extract_exec(p, 5);
2315
2316         /* Thread. */
2317         thread_ip_inc(p);
2318 }
2319
2320 static inline void
2321 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2322 {
2323         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2324               p->thread_id);
2325
2326         __instr_hdr_extract_exec(p, 6);
2327
2328         /* Thread. */
2329         thread_ip_inc(p);
2330 }
2331
2332 static inline void
2333 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2334 {
2335         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2336               p->thread_id);
2337
2338         __instr_hdr_extract_exec(p, 7);
2339
2340         /* Thread. */
2341         thread_ip_inc(p);
2342 }
2343
2344 static inline void
2345 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2346 {
2347         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2348               p->thread_id);
2349
2350         __instr_hdr_extract_exec(p, 8);
2351
2352         /* Thread. */
2353         thread_ip_inc(p);
2354 }
2355
2356 /*
2357  * emit.
2358  */
2359 static int
2360 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2361                          struct action *action __rte_unused,
2362                          char **tokens,
2363                          int n_tokens,
2364                          struct instruction *instr,
2365                          struct instruction_data *data __rte_unused)
2366 {
2367         struct header *h;
2368
2369         CHECK(n_tokens == 2, EINVAL);
2370
2371         h = header_parse(p, tokens[1]);
2372         CHECK(h, EINVAL);
2373
2374         instr->type = INSTR_HDR_EMIT;
2375         instr->io.hdr.header_id[0] = h->id;
2376         instr->io.hdr.struct_id[0] = h->struct_id;
2377         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2378         return 0;
2379 }
2380
2381 static inline void
2382 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
2383
2384 static inline void
2385 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
2386 {
2387         struct thread *t = &p->threads[p->thread_id];
2388         struct instruction *ip = t->ip;
2389         uint32_t n_headers_out = t->n_headers_out;
2390         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
2391         uint8_t *ho_ptr = NULL;
2392         uint32_t ho_nbytes = 0, i;
2393
2394         for (i = 0; i < n_emit; i++) {
2395                 uint32_t header_id = ip->io.hdr.header_id[i];
2396                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2397                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2398
2399                 struct header_runtime *hi = &t->headers[header_id];
2400                 uint8_t *hi_ptr = t->structs[struct_id];
2401
2402                 TRACE("[Thread %2u]: emit header %u\n",
2403                       p->thread_id,
2404                       header_id);
2405
2406                 /* Headers. */
2407                 if (!i) {
2408                         if (!t->n_headers_out) {
2409                                 ho = &t->headers_out[0];
2410
2411                                 ho->ptr0 = hi->ptr0;
2412                                 ho->ptr = hi_ptr;
2413
2414                                 ho_ptr = hi_ptr;
2415                                 ho_nbytes = n_bytes;
2416
2417                                 n_headers_out = 1;
2418
2419                                 continue;
2420                         } else {
2421                                 ho_ptr = ho->ptr;
2422                                 ho_nbytes = ho->n_bytes;
2423                         }
2424                 }
2425
2426                 if (ho_ptr + ho_nbytes == hi_ptr) {
2427                         ho_nbytes += n_bytes;
2428                 } else {
2429                         ho->n_bytes = ho_nbytes;
2430
2431                         ho++;
2432                         ho->ptr0 = hi->ptr0;
2433                         ho->ptr = hi_ptr;
2434
2435                         ho_ptr = hi_ptr;
2436                         ho_nbytes = n_bytes;
2437
2438                         n_headers_out++;
2439                 }
2440         }
2441
2442         ho->n_bytes = ho_nbytes;
2443         t->n_headers_out = n_headers_out;
2444 }
2445
2446 static inline void
2447 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2448 {
2449         __instr_hdr_emit_exec(p, 1);
2450
2451         /* Thread. */
2452         thread_ip_inc(p);
2453 }
2454
2455 static inline void
2456 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2457 {
2458         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2459               p->thread_id);
2460
2461         __instr_hdr_emit_exec(p, 1);
2462         instr_tx_exec(p);
2463 }
2464
2465 static inline void
2466 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2467 {
2468         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2469               p->thread_id);
2470
2471         __instr_hdr_emit_exec(p, 2);
2472         instr_tx_exec(p);
2473 }
2474
2475 static inline void
2476 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2477 {
2478         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2479               p->thread_id);
2480
2481         __instr_hdr_emit_exec(p, 3);
2482         instr_tx_exec(p);
2483 }
2484
2485 static inline void
2486 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2487 {
2488         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2489               p->thread_id);
2490
2491         __instr_hdr_emit_exec(p, 4);
2492         instr_tx_exec(p);
2493 }
2494
2495 static inline void
2496 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2497 {
2498         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2499               p->thread_id);
2500
2501         __instr_hdr_emit_exec(p, 5);
2502         instr_tx_exec(p);
2503 }
2504
2505 static inline void
2506 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2507 {
2508         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2509               p->thread_id);
2510
2511         __instr_hdr_emit_exec(p, 6);
2512         instr_tx_exec(p);
2513 }
2514
2515 static inline void
2516 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2517 {
2518         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2519               p->thread_id);
2520
2521         __instr_hdr_emit_exec(p, 7);
2522         instr_tx_exec(p);
2523 }
2524
2525 static inline void
2526 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2527 {
2528         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2529               p->thread_id);
2530
2531         __instr_hdr_emit_exec(p, 8);
2532         instr_tx_exec(p);
2533 }
2534
2535 /*
2536  * validate.
2537  */
2538 static int
2539 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2540                              struct action *action __rte_unused,
2541                              char **tokens,
2542                              int n_tokens,
2543                              struct instruction *instr,
2544                              struct instruction_data *data __rte_unused)
2545 {
2546         struct header *h;
2547
2548         CHECK(n_tokens == 2, EINVAL);
2549
2550         h = header_parse(p, tokens[1]);
2551         CHECK(h, EINVAL);
2552
2553         instr->type = INSTR_HDR_VALIDATE;
2554         instr->valid.header_id = h->id;
2555         return 0;
2556 }
2557
2558 static inline void
2559 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2560 {
2561         struct thread *t = &p->threads[p->thread_id];
2562         struct instruction *ip = t->ip;
2563         uint32_t header_id = ip->valid.header_id;
2564
2565         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2566
2567         /* Headers. */
2568         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2569
2570         /* Thread. */
2571         thread_ip_inc(p);
2572 }
2573
2574 /*
2575  * invalidate.
2576  */
2577 static int
2578 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2579                                struct action *action __rte_unused,
2580                                char **tokens,
2581                                int n_tokens,
2582                                struct instruction *instr,
2583                                struct instruction_data *data __rte_unused)
2584 {
2585         struct header *h;
2586
2587         CHECK(n_tokens == 2, EINVAL);
2588
2589         h = header_parse(p, tokens[1]);
2590         CHECK(h, EINVAL);
2591
2592         instr->type = INSTR_HDR_INVALIDATE;
2593         instr->valid.header_id = h->id;
2594         return 0;
2595 }
2596
2597 static inline void
2598 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2599 {
2600         struct thread *t = &p->threads[p->thread_id];
2601         struct instruction *ip = t->ip;
2602         uint32_t header_id = ip->valid.header_id;
2603
2604         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2605
2606         /* Headers. */
2607         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2608
2609         /* Thread. */
2610         thread_ip_inc(p);
2611 }
2612
2613 /*
2614  * mov.
2615  */
2616 static int
2617 instr_mov_translate(struct rte_swx_pipeline *p,
2618                     struct action *action,
2619                     char **tokens,
2620                     int n_tokens,
2621                     struct instruction *instr,
2622                     struct instruction_data *data __rte_unused)
2623 {
2624         char *dst = tokens[1], *src = tokens[2];
2625         struct field *fdst, *fsrc;
2626         uint32_t dst_struct_id, src_struct_id, src_val;
2627
2628         CHECK(n_tokens == 3, EINVAL);
2629
2630         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2631         CHECK(fdst, EINVAL);
2632
2633         /* MOV or MOV_S. */
2634         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2635         if (fsrc) {
2636                 instr->type = INSTR_MOV;
2637                 if ((dst[0] == 'h' && src[0] != 'h') ||
2638                     (dst[0] != 'h' && src[0] == 'h'))
2639                         instr->type = INSTR_MOV_S;
2640
2641                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2642                 instr->mov.dst.n_bits = fdst->n_bits;
2643                 instr->mov.dst.offset = fdst->offset / 8;
2644                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2645                 instr->mov.src.n_bits = fsrc->n_bits;
2646                 instr->mov.src.offset = fsrc->offset / 8;
2647                 return 0;
2648         }
2649
2650         /* MOV_I. */
2651         src_val = strtoul(src, &src, 0);
2652         CHECK(!src[0], EINVAL);
2653
2654         if (dst[0] == 'h')
2655                 src_val = htonl(src_val);
2656
2657         instr->type = INSTR_MOV_I;
2658         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2659         instr->mov.dst.n_bits = fdst->n_bits;
2660         instr->mov.dst.offset = fdst->offset / 8;
2661         instr->mov.src_val = (uint32_t)src_val;
2662         return 0;
2663 }
2664
2665 static inline void
2666 instr_mov_exec(struct rte_swx_pipeline *p)
2667 {
2668         struct thread *t = &p->threads[p->thread_id];
2669         struct instruction *ip = t->ip;
2670
2671         TRACE("[Thread %2u] mov\n",
2672               p->thread_id);
2673
2674         MOV(t, ip);
2675
2676         /* Thread. */
2677         thread_ip_inc(p);
2678 }
2679
2680 static inline void
2681 instr_mov_s_exec(struct rte_swx_pipeline *p)
2682 {
2683         struct thread *t = &p->threads[p->thread_id];
2684         struct instruction *ip = t->ip;
2685
2686         TRACE("[Thread %2u] mov (s)\n",
2687               p->thread_id);
2688
2689         MOV_S(t, ip);
2690
2691         /* Thread. */
2692         thread_ip_inc(p);
2693 }
2694
2695 static inline void
2696 instr_mov_i_exec(struct rte_swx_pipeline *p)
2697 {
2698         struct thread *t = &p->threads[p->thread_id];
2699         struct instruction *ip = t->ip;
2700
2701         TRACE("[Thread %2u] mov m.f %x\n",
2702               p->thread_id,
2703               ip->mov.src_val);
2704
2705         MOV_I(t, ip);
2706
2707         /* Thread. */
2708         thread_ip_inc(p);
2709 }
2710
2711 /*
2712  * dma.
2713  */
2714 static int
2715 instr_dma_translate(struct rte_swx_pipeline *p,
2716                     struct action *action,
2717                     char **tokens,
2718                     int n_tokens,
2719                     struct instruction *instr,
2720                     struct instruction_data *data __rte_unused)
2721 {
2722         char *dst = tokens[1];
2723         char *src = tokens[2];
2724         struct header *h;
2725         struct field *tf;
2726
2727         CHECK(action, EINVAL);
2728         CHECK(n_tokens == 3, EINVAL);
2729
2730         h = header_parse(p, dst);
2731         CHECK(h, EINVAL);
2732
2733         tf = action_field_parse(action, src);
2734         CHECK(tf, EINVAL);
2735
2736         instr->type = INSTR_DMA_HT;
2737         instr->dma.dst.header_id[0] = h->id;
2738         instr->dma.dst.struct_id[0] = h->struct_id;
2739         instr->dma.n_bytes[0] = h->st->n_bits / 8;
2740         instr->dma.src.offset[0] = tf->offset / 8;
2741
2742         return 0;
2743 }
2744
2745 static inline void
2746 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2747
2748 static inline void
2749 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2750 {
2751         struct thread *t = &p->threads[p->thread_id];
2752         struct instruction *ip = t->ip;
2753         uint8_t *action_data = t->structs[0];
2754         uint64_t valid_headers = t->valid_headers;
2755         uint32_t i;
2756
2757         for (i = 0; i < n_dma; i++) {
2758                 uint32_t header_id = ip->dma.dst.header_id[i];
2759                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2760                 uint32_t offset = ip->dma.src.offset[i];
2761                 uint32_t n_bytes = ip->dma.n_bytes[i];
2762
2763                 struct header_runtime *h = &t->headers[header_id];
2764                 uint8_t *h_ptr0 = h->ptr0;
2765                 uint8_t *h_ptr = t->structs[struct_id];
2766
2767                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2768                         h_ptr : h_ptr0;
2769                 void *src = &action_data[offset];
2770
2771                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2772
2773                 /* Headers. */
2774                 memcpy(dst, src, n_bytes);
2775                 t->structs[struct_id] = dst;
2776                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2777         }
2778
2779         t->valid_headers = valid_headers;
2780 }
2781
2782 static inline void
2783 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2784 {
2785         __instr_dma_ht_exec(p, 1);
2786
2787         /* Thread. */
2788         thread_ip_inc(p);
2789 }
2790
2791 static inline void
2792 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2793 {
2794         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2795               p->thread_id);
2796
2797         __instr_dma_ht_exec(p, 2);
2798
2799         /* Thread. */
2800         thread_ip_inc(p);
2801 }
2802
2803 static inline void
2804 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2805 {
2806         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2807               p->thread_id);
2808
2809         __instr_dma_ht_exec(p, 3);
2810
2811         /* Thread. */
2812         thread_ip_inc(p);
2813 }
2814
2815 static inline void
2816 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2817 {
2818         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2819               p->thread_id);
2820
2821         __instr_dma_ht_exec(p, 4);
2822
2823         /* Thread. */
2824         thread_ip_inc(p);
2825 }
2826
2827 static inline void
2828 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2829 {
2830         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2831               p->thread_id);
2832
2833         __instr_dma_ht_exec(p, 5);
2834
2835         /* Thread. */
2836         thread_ip_inc(p);
2837 }
2838
2839 static inline void
2840 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2841 {
2842         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2843               p->thread_id);
2844
2845         __instr_dma_ht_exec(p, 6);
2846
2847         /* Thread. */
2848         thread_ip_inc(p);
2849 }
2850
2851 static inline void
2852 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2853 {
2854         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2855               p->thread_id);
2856
2857         __instr_dma_ht_exec(p, 7);
2858
2859         /* Thread. */
2860         thread_ip_inc(p);
2861 }
2862
2863 static inline void
2864 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2865 {
2866         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2867               p->thread_id);
2868
2869         __instr_dma_ht_exec(p, 8);
2870
2871         /* Thread. */
2872         thread_ip_inc(p);
2873 }
2874
2875 /*
2876  * alu.
2877  */
2878 static int
2879 instr_alu_add_translate(struct rte_swx_pipeline *p,
2880                         struct action *action,
2881                         char **tokens,
2882                         int n_tokens,
2883                         struct instruction *instr,
2884                         struct instruction_data *data __rte_unused)
2885 {
2886         char *dst = tokens[1], *src = tokens[2];
2887         struct field *fdst, *fsrc;
2888         uint32_t dst_struct_id, src_struct_id, src_val;
2889
2890         CHECK(n_tokens == 3, EINVAL);
2891
2892         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2893         CHECK(fdst, EINVAL);
2894
2895         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2896         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2897         if (fsrc) {
2898                 instr->type = INSTR_ALU_ADD;
2899                 if (dst[0] == 'h' && src[0] == 'm')
2900                         instr->type = INSTR_ALU_ADD_HM;
2901                 if (dst[0] == 'm' && src[0] == 'h')
2902                         instr->type = INSTR_ALU_ADD_MH;
2903                 if (dst[0] == 'h' && src[0] == 'h')
2904                         instr->type = INSTR_ALU_ADD_HH;
2905
2906                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2907                 instr->alu.dst.n_bits = fdst->n_bits;
2908                 instr->alu.dst.offset = fdst->offset / 8;
2909                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2910                 instr->alu.src.n_bits = fsrc->n_bits;
2911                 instr->alu.src.offset = fsrc->offset / 8;
2912                 return 0;
2913         }
2914
2915         /* ADD_MI, ADD_HI. */
2916         src_val = strtoul(src, &src, 0);
2917         CHECK(!src[0], EINVAL);
2918
2919         instr->type = INSTR_ALU_ADD_MI;
2920         if (dst[0] == 'h')
2921                 instr->type = INSTR_ALU_ADD_HI;
2922
2923         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2924         instr->alu.dst.n_bits = fdst->n_bits;
2925         instr->alu.dst.offset = fdst->offset / 8;
2926         instr->alu.src_val = (uint32_t)src_val;
2927         return 0;
2928 }
2929
2930 static int
2931 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2932                         struct action *action,
2933                         char **tokens,
2934                         int n_tokens,
2935                         struct instruction *instr,
2936                         struct instruction_data *data __rte_unused)
2937 {
2938         char *dst = tokens[1], *src = tokens[2];
2939         struct field *fdst, *fsrc;
2940         uint32_t dst_struct_id, src_struct_id, src_val;
2941
2942         CHECK(n_tokens == 3, EINVAL);
2943
2944         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2945         CHECK(fdst, EINVAL);
2946
2947         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2948         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2949         if (fsrc) {
2950                 instr->type = INSTR_ALU_SUB;
2951                 if (dst[0] == 'h' && src[0] == 'm')
2952                         instr->type = INSTR_ALU_SUB_HM;
2953                 if (dst[0] == 'm' && src[0] == 'h')
2954                         instr->type = INSTR_ALU_SUB_MH;
2955                 if (dst[0] == 'h' && src[0] == 'h')
2956                         instr->type = INSTR_ALU_SUB_HH;
2957
2958                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2959                 instr->alu.dst.n_bits = fdst->n_bits;
2960                 instr->alu.dst.offset = fdst->offset / 8;
2961                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2962                 instr->alu.src.n_bits = fsrc->n_bits;
2963                 instr->alu.src.offset = fsrc->offset / 8;
2964                 return 0;
2965         }
2966
2967         /* SUB_MI, SUB_HI. */
2968         src_val = strtoul(src, &src, 0);
2969         CHECK(!src[0], EINVAL);
2970
2971         instr->type = INSTR_ALU_SUB_MI;
2972         if (dst[0] == 'h')
2973                 instr->type = INSTR_ALU_SUB_HI;
2974
2975         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2976         instr->alu.dst.n_bits = fdst->n_bits;
2977         instr->alu.dst.offset = fdst->offset / 8;
2978         instr->alu.src_val = (uint32_t)src_val;
2979         return 0;
2980 }
2981
2982 static inline void
2983 instr_alu_add_exec(struct rte_swx_pipeline *p)
2984 {
2985         struct thread *t = &p->threads[p->thread_id];
2986         struct instruction *ip = t->ip;
2987
2988         TRACE("[Thread %2u] add\n", p->thread_id);
2989
2990         /* Structs. */
2991         ALU(t, ip, +);
2992
2993         /* Thread. */
2994         thread_ip_inc(p);
2995 }
2996
2997 static inline void
2998 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
2999 {
3000         struct thread *t = &p->threads[p->thread_id];
3001         struct instruction *ip = t->ip;
3002
3003         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3004
3005         /* Structs. */
3006         ALU_MH(t, ip, +);
3007
3008         /* Thread. */
3009         thread_ip_inc(p);
3010 }
3011
3012 static inline void
3013 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3014 {
3015         struct thread *t = &p->threads[p->thread_id];
3016         struct instruction *ip = t->ip;
3017
3018         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3019
3020         /* Structs. */
3021         ALU_HM(t, ip, +);
3022
3023         /* Thread. */
3024         thread_ip_inc(p);
3025 }
3026
3027 static inline void
3028 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3029 {
3030         struct thread *t = &p->threads[p->thread_id];
3031         struct instruction *ip = t->ip;
3032
3033         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3034
3035         /* Structs. */
3036         ALU_HH(t, ip, +);
3037
3038         /* Thread. */
3039         thread_ip_inc(p);
3040 }
3041
3042 static inline void
3043 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3044 {
3045         struct thread *t = &p->threads[p->thread_id];
3046         struct instruction *ip = t->ip;
3047
3048         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3049
3050         /* Structs. */
3051         ALU_MI(t, ip, +);
3052
3053         /* Thread. */
3054         thread_ip_inc(p);
3055 }
3056
3057 static inline void
3058 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3059 {
3060         struct thread *t = &p->threads[p->thread_id];
3061         struct instruction *ip = t->ip;
3062
3063         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3064
3065         /* Structs. */
3066         ALU_HI(t, ip, +);
3067
3068         /* Thread. */
3069         thread_ip_inc(p);
3070 }
3071
3072 static inline void
3073 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3074 {
3075         struct thread *t = &p->threads[p->thread_id];
3076         struct instruction *ip = t->ip;
3077
3078         TRACE("[Thread %2u] sub\n", p->thread_id);
3079
3080         /* Structs. */
3081         ALU(t, ip, -);
3082
3083         /* Thread. */
3084         thread_ip_inc(p);
3085 }
3086
3087 static inline void
3088 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3089 {
3090         struct thread *t = &p->threads[p->thread_id];
3091         struct instruction *ip = t->ip;
3092
3093         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3094
3095         /* Structs. */
3096         ALU_MH(t, ip, -);
3097
3098         /* Thread. */
3099         thread_ip_inc(p);
3100 }
3101
3102 static inline void
3103 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3104 {
3105         struct thread *t = &p->threads[p->thread_id];
3106         struct instruction *ip = t->ip;
3107
3108         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3109
3110         /* Structs. */
3111         ALU_HM(t, ip, -);
3112
3113         /* Thread. */
3114         thread_ip_inc(p);
3115 }
3116
3117 static inline void
3118 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3119 {
3120         struct thread *t = &p->threads[p->thread_id];
3121         struct instruction *ip = t->ip;
3122
3123         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3124
3125         /* Structs. */
3126         ALU_HH(t, ip, -);
3127
3128         /* Thread. */
3129         thread_ip_inc(p);
3130 }
3131
3132 static inline void
3133 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3134 {
3135         struct thread *t = &p->threads[p->thread_id];
3136         struct instruction *ip = t->ip;
3137
3138         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3139
3140         /* Structs. */
3141         ALU_MI(t, ip, -);
3142
3143         /* Thread. */
3144         thread_ip_inc(p);
3145 }
3146
3147 static inline void
3148 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3149 {
3150         struct thread *t = &p->threads[p->thread_id];
3151         struct instruction *ip = t->ip;
3152
3153         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3154
3155         /* Structs. */
3156         ALU_HI(t, ip, -);
3157
3158         /* Thread. */
3159         thread_ip_inc(p);
3160 }
3161
3162 #define RTE_SWX_INSTRUCTION_TOKENS_MAX 16
3163
3164 static int
3165 instr_translate(struct rte_swx_pipeline *p,
3166                 struct action *action,
3167                 char *string,
3168                 struct instruction *instr,
3169                 struct instruction_data *data)
3170 {
3171         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
3172         int n_tokens = 0, tpos = 0;
3173
3174         /* Parse the instruction string into tokens. */
3175         for ( ; ; ) {
3176                 char *token;
3177
3178                 token = strtok_r(string, " \t\v", &string);
3179                 if (!token)
3180                         break;
3181
3182                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
3183
3184                 tokens[n_tokens] = token;
3185                 n_tokens++;
3186         }
3187
3188         CHECK(n_tokens, EINVAL);
3189
3190         /* Handle the optional instruction label. */
3191         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
3192                 strcpy(data->label, tokens[0]);
3193
3194                 tpos += 2;
3195                 CHECK(n_tokens - tpos, EINVAL);
3196         }
3197
3198         /* Identify the instruction type. */
3199         if (!strcmp(tokens[tpos], "rx"))
3200                 return instr_rx_translate(p,
3201                                           action,
3202                                           &tokens[tpos],
3203                                           n_tokens - tpos,
3204                                           instr,
3205                                           data);
3206
3207         if (!strcmp(tokens[tpos], "tx"))
3208                 return instr_tx_translate(p,
3209                                           action,
3210                                           &tokens[tpos],
3211                                           n_tokens - tpos,
3212                                           instr,
3213                                           data);
3214
3215         if (!strcmp(tokens[tpos], "extract"))
3216                 return instr_hdr_extract_translate(p,
3217                                                    action,
3218                                                    &tokens[tpos],
3219                                                    n_tokens - tpos,
3220                                                    instr,
3221                                                    data);
3222
3223         if (!strcmp(tokens[tpos], "emit"))
3224                 return instr_hdr_emit_translate(p,
3225                                                 action,
3226                                                 &tokens[tpos],
3227                                                 n_tokens - tpos,
3228                                                 instr,
3229                                                 data);
3230
3231         if (!strcmp(tokens[tpos], "validate"))
3232                 return instr_hdr_validate_translate(p,
3233                                                     action,
3234                                                     &tokens[tpos],
3235                                                     n_tokens - tpos,
3236                                                     instr,
3237                                                     data);
3238
3239         if (!strcmp(tokens[tpos], "invalidate"))
3240                 return instr_hdr_invalidate_translate(p,
3241                                                       action,
3242                                                       &tokens[tpos],
3243                                                       n_tokens - tpos,
3244                                                       instr,
3245                                                       data);
3246
3247         if (!strcmp(tokens[tpos], "mov"))
3248                 return instr_mov_translate(p,
3249                                            action,
3250                                            &tokens[tpos],
3251                                            n_tokens - tpos,
3252                                            instr,
3253                                            data);
3254
3255         if (!strcmp(tokens[tpos], "dma"))
3256                 return instr_dma_translate(p,
3257                                            action,
3258                                            &tokens[tpos],
3259                                            n_tokens - tpos,
3260                                            instr,
3261                                            data);
3262
3263         if (!strcmp(tokens[tpos], "add"))
3264                 return instr_alu_add_translate(p,
3265                                                action,
3266                                                &tokens[tpos],
3267                                                n_tokens - tpos,
3268                                                instr,
3269                                                data);
3270
3271         if (!strcmp(tokens[tpos], "sub"))
3272                 return instr_alu_sub_translate(p,
3273                                                action,
3274                                                &tokens[tpos],
3275                                                n_tokens - tpos,
3276                                                instr,
3277                                                data);
3278
3279         CHECK(0, EINVAL);
3280 }
3281
3282 static uint32_t
3283 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
3284 {
3285         uint32_t count = 0, i;
3286
3287         if (!label[0])
3288                 return 0;
3289
3290         for (i = 0; i < n; i++)
3291                 if (!strcmp(label, data[i].jmp_label))
3292                         count++;
3293
3294         return count;
3295 }
3296
3297 static int
3298 instr_label_check(struct instruction_data *instruction_data,
3299                   uint32_t n_instructions)
3300 {
3301         uint32_t i;
3302
3303         /* Check that all instruction labels are unique. */
3304         for (i = 0; i < n_instructions; i++) {
3305                 struct instruction_data *data = &instruction_data[i];
3306                 char *label = data->label;
3307                 uint32_t j;
3308
3309                 if (!label[0])
3310                         continue;
3311
3312                 for (j = i + 1; j < n_instructions; j++)
3313                         CHECK(strcmp(label, data[j].label), EINVAL);
3314         }
3315
3316         /* Get users for each instruction label. */
3317         for (i = 0; i < n_instructions; i++) {
3318                 struct instruction_data *data = &instruction_data[i];
3319                 char *label = data->label;
3320
3321                 data->n_users = label_is_used(instruction_data,
3322                                               n_instructions,
3323                                               label);
3324         }
3325
3326         return 0;
3327 }
3328
3329 static int
3330 instruction_config(struct rte_swx_pipeline *p,
3331                    struct action *a,
3332                    const char **instructions,
3333                    uint32_t n_instructions)
3334 {
3335         struct instruction *instr = NULL;
3336         struct instruction_data *data = NULL;
3337         char *string = NULL;
3338         int err = 0;
3339         uint32_t i;
3340
3341         CHECK(n_instructions, EINVAL);
3342         CHECK(instructions, EINVAL);
3343         for (i = 0; i < n_instructions; i++)
3344                 CHECK(instructions[i], EINVAL);
3345
3346         /* Memory allocation. */
3347         instr = calloc(n_instructions, sizeof(struct instruction));
3348         if (!instr) {
3349                 err = ENOMEM;
3350                 goto error;
3351         }
3352
3353         data = calloc(n_instructions, sizeof(struct instruction_data));
3354         if (!data) {
3355                 err = ENOMEM;
3356                 goto error;
3357         }
3358
3359         for (i = 0; i < n_instructions; i++) {
3360                 string = strdup(instructions[i]);
3361                 if (!string) {
3362                         err = ENOMEM;
3363                         goto error;
3364                 }
3365
3366                 err = instr_translate(p, a, string, &instr[i], &data[i]);
3367                 if (err)
3368                         goto error;
3369
3370                 free(string);
3371         }
3372
3373         err = instr_label_check(data, n_instructions);
3374         if (err)
3375                 goto error;
3376
3377         free(data);
3378
3379         if (a) {
3380                 a->instructions = instr;
3381                 a->n_instructions = n_instructions;
3382         } else {
3383                 p->instructions = instr;
3384                 p->n_instructions = n_instructions;
3385         }
3386
3387         return 0;
3388
3389 error:
3390         free(string);
3391         free(data);
3392         free(instr);
3393         return err;
3394 }
3395
3396 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
3397
3398 static instr_exec_t instruction_table[] = {
3399         [INSTR_RX] = instr_rx_exec,
3400         [INSTR_TX] = instr_tx_exec,
3401
3402         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
3403         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
3404         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
3405         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
3406         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
3407         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
3408         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
3409         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
3410
3411         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
3412         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
3413         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
3414         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
3415         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
3416         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
3417         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
3418         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
3419         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
3420
3421         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
3422         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
3423
3424         [INSTR_MOV] = instr_mov_exec,
3425         [INSTR_MOV_S] = instr_mov_s_exec,
3426         [INSTR_MOV_I] = instr_mov_i_exec,
3427
3428         [INSTR_DMA_HT] = instr_dma_ht_exec,
3429         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
3430         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
3431         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
3432         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
3433         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
3434         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
3435         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
3436
3437         [INSTR_ALU_ADD] = instr_alu_add_exec,
3438         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
3439         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
3440         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
3441         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
3442         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
3443
3444         [INSTR_ALU_SUB] = instr_alu_sub_exec,
3445         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
3446         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
3447         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
3448         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
3449         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
3450 };
3451
3452 static inline void
3453 instr_exec(struct rte_swx_pipeline *p)
3454 {
3455         struct thread *t = &p->threads[p->thread_id];
3456         struct instruction *ip = t->ip;
3457         instr_exec_t instr = instruction_table[ip->type];
3458
3459         instr(p);
3460 }
3461
3462 /*
3463  * Action.
3464  */
3465 static struct action *
3466 action_find(struct rte_swx_pipeline *p, const char *name)
3467 {
3468         struct action *elem;
3469
3470         if (!name)
3471                 return NULL;
3472
3473         TAILQ_FOREACH(elem, &p->actions, node)
3474                 if (strcmp(elem->name, name) == 0)
3475                         return elem;
3476
3477         return NULL;
3478 }
3479
3480 static struct field *
3481 action_field_find(struct action *a, const char *name)
3482 {
3483         return a->st ? struct_type_field_find(a->st, name) : NULL;
3484 }
3485
3486 static struct field *
3487 action_field_parse(struct action *action, const char *name)
3488 {
3489         if (name[0] != 't' || name[1] != '.')
3490                 return NULL;
3491
3492         return action_field_find(action, &name[2]);
3493 }
3494
3495 int
3496 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
3497                                const char *name,
3498                                const char *args_struct_type_name,
3499                                const char **instructions,
3500                                uint32_t n_instructions)
3501 {
3502         struct struct_type *args_struct_type;
3503         struct action *a;
3504         int err;
3505
3506         CHECK(p, EINVAL);
3507
3508         CHECK_NAME(name, EINVAL);
3509         CHECK(!action_find(p, name), EEXIST);
3510
3511         if (args_struct_type_name) {
3512                 CHECK_NAME(args_struct_type_name, EINVAL);
3513                 args_struct_type = struct_type_find(p, args_struct_type_name);
3514                 CHECK(args_struct_type, EINVAL);
3515         } else {
3516                 args_struct_type = NULL;
3517         }
3518
3519         /* Node allocation. */
3520         a = calloc(1, sizeof(struct action));
3521         CHECK(a, ENOMEM);
3522
3523         /* Node initialization. */
3524         strcpy(a->name, name);
3525         a->st = args_struct_type;
3526         a->id = p->n_actions;
3527
3528         /* Instruction translation. */
3529         err = instruction_config(p, a, instructions, n_instructions);
3530         if (err) {
3531                 free(a);
3532                 return err;
3533         }
3534
3535         /* Node add to tailq. */
3536         TAILQ_INSERT_TAIL(&p->actions, a, node);
3537         p->n_actions++;
3538
3539         return 0;
3540 }
3541
3542 static int
3543 action_build(struct rte_swx_pipeline *p)
3544 {
3545         struct action *action;
3546
3547         p->action_instructions = calloc(p->n_actions,
3548                                         sizeof(struct instruction *));
3549         CHECK(p->action_instructions, ENOMEM);
3550
3551         TAILQ_FOREACH(action, &p->actions, node)
3552                 p->action_instructions[action->id] = action->instructions;
3553
3554         return 0;
3555 }
3556
3557 static void
3558 action_build_free(struct rte_swx_pipeline *p)
3559 {
3560         free(p->action_instructions);
3561         p->action_instructions = NULL;
3562 }
3563
3564 static void
3565 action_free(struct rte_swx_pipeline *p)
3566 {
3567         action_build_free(p);
3568
3569         for ( ; ; ) {
3570                 struct action *action;
3571
3572                 action = TAILQ_FIRST(&p->actions);
3573                 if (!action)
3574                         break;
3575
3576                 TAILQ_REMOVE(&p->actions, action, node);
3577                 free(action->instructions);
3578                 free(action);
3579         }
3580 }
3581
3582 /*
3583  * Table.
3584  */
3585 static struct table_type *
3586 table_type_find(struct rte_swx_pipeline *p, const char *name)
3587 {
3588         struct table_type *elem;
3589
3590         TAILQ_FOREACH(elem, &p->table_types, node)
3591                 if (strcmp(elem->name, name) == 0)
3592                         return elem;
3593
3594         return NULL;
3595 }
3596
3597 static struct table_type *
3598 table_type_resolve(struct rte_swx_pipeline *p,
3599                    const char *recommended_type_name,
3600                    enum rte_swx_table_match_type match_type)
3601 {
3602         struct table_type *elem;
3603
3604         /* Only consider the recommended type if the match type is correct. */
3605         if (recommended_type_name)
3606                 TAILQ_FOREACH(elem, &p->table_types, node)
3607                         if (!strcmp(elem->name, recommended_type_name) &&
3608                             (elem->match_type == match_type))
3609                                 return elem;
3610
3611         /* Ignore the recommended type and get the first element with this match
3612          * type.
3613          */
3614         TAILQ_FOREACH(elem, &p->table_types, node)
3615                 if (elem->match_type == match_type)
3616                         return elem;
3617
3618         return NULL;
3619 }
3620
3621 static struct table *
3622 table_find(struct rte_swx_pipeline *p, const char *name)
3623 {
3624         struct table *elem;
3625
3626         TAILQ_FOREACH(elem, &p->tables, node)
3627                 if (strcmp(elem->name, name) == 0)
3628                         return elem;
3629
3630         return NULL;
3631 }
3632
3633 static struct table *
3634 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
3635 {
3636         struct table *table = NULL;
3637
3638         TAILQ_FOREACH(table, &p->tables, node)
3639                 if (table->id == id)
3640                         return table;
3641
3642         return NULL;
3643 }
3644
3645 int
3646 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
3647                                      const char *name,
3648                                      enum rte_swx_table_match_type match_type,
3649                                      struct rte_swx_table_ops *ops)
3650 {
3651         struct table_type *elem;
3652
3653         CHECK(p, EINVAL);
3654
3655         CHECK_NAME(name, EINVAL);
3656         CHECK(!table_type_find(p, name), EEXIST);
3657
3658         CHECK(ops, EINVAL);
3659         CHECK(ops->create, EINVAL);
3660         CHECK(ops->lkp, EINVAL);
3661         CHECK(ops->free, EINVAL);
3662
3663         /* Node allocation. */
3664         elem = calloc(1, sizeof(struct table_type));
3665         CHECK(elem, ENOMEM);
3666
3667         /* Node initialization. */
3668         strcpy(elem->name, name);
3669         elem->match_type = match_type;
3670         memcpy(&elem->ops, ops, sizeof(*ops));
3671
3672         /* Node add to tailq. */
3673         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
3674
3675         return 0;
3676 }
3677
3678 static enum rte_swx_table_match_type
3679 table_match_type_resolve(struct rte_swx_match_field_params *fields,
3680                          uint32_t n_fields)
3681 {
3682         uint32_t i;
3683
3684         for (i = 0; i < n_fields; i++)
3685                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
3686                         break;
3687
3688         if (i == n_fields)
3689                 return RTE_SWX_TABLE_MATCH_EXACT;
3690
3691         if ((i == n_fields - 1) &&
3692             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
3693                 return RTE_SWX_TABLE_MATCH_LPM;
3694
3695         return RTE_SWX_TABLE_MATCH_WILDCARD;
3696 }
3697
3698 int
3699 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
3700                               const char *name,
3701                               struct rte_swx_pipeline_table_params *params,
3702                               const char *recommended_table_type_name,
3703                               const char *args,
3704                               uint32_t size)
3705 {
3706         struct table_type *type;
3707         struct table *t;
3708         struct action *default_action;
3709         struct header *header = NULL;
3710         int is_header = 0;
3711         uint32_t offset_prev = 0, action_data_size_max = 0, i;
3712
3713         CHECK(p, EINVAL);
3714
3715         CHECK_NAME(name, EINVAL);
3716         CHECK(!table_find(p, name), EEXIST);
3717
3718         CHECK(params, EINVAL);
3719
3720         /* Match checks. */
3721         CHECK(!params->n_fields || params->fields, EINVAL);
3722         for (i = 0; i < params->n_fields; i++) {
3723                 struct rte_swx_match_field_params *field = &params->fields[i];
3724                 struct header *h;
3725                 struct field *hf, *mf;
3726                 uint32_t offset;
3727
3728                 CHECK_NAME(field->name, EINVAL);
3729
3730                 hf = header_field_parse(p, field->name, &h);
3731                 mf = metadata_field_parse(p, field->name);
3732                 CHECK(hf || mf, EINVAL);
3733
3734                 offset = hf ? hf->offset : mf->offset;
3735
3736                 if (i == 0) {
3737                         is_header = hf ? 1 : 0;
3738                         header = hf ? h : NULL;
3739                         offset_prev = offset;
3740
3741                         continue;
3742                 }
3743
3744                 CHECK((is_header && hf && (h->id == header->id)) ||
3745                       (!is_header && mf), EINVAL);
3746
3747                 CHECK(offset > offset_prev, EINVAL);
3748                 offset_prev = offset;
3749         }
3750
3751         /* Action checks. */
3752         CHECK(params->n_actions, EINVAL);
3753         CHECK(params->action_names, EINVAL);
3754         for (i = 0; i < params->n_actions; i++) {
3755                 const char *action_name = params->action_names[i];
3756                 struct action *a;
3757                 uint32_t action_data_size;
3758
3759                 CHECK(action_name, EINVAL);
3760
3761                 a = action_find(p, action_name);
3762                 CHECK(a, EINVAL);
3763
3764                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
3765                 if (action_data_size > action_data_size_max)
3766                         action_data_size_max = action_data_size;
3767         }
3768
3769         CHECK(params->default_action_name, EINVAL);
3770         for (i = 0; i < p->n_actions; i++)
3771                 if (!strcmp(params->action_names[i],
3772                             params->default_action_name))
3773                         break;
3774         CHECK(i < params->n_actions, EINVAL);
3775         default_action = action_find(p, params->default_action_name);
3776         CHECK((default_action->st && params->default_action_data) ||
3777               !params->default_action_data, EINVAL);
3778
3779         /* Table type checks. */
3780         if (params->n_fields) {
3781                 enum rte_swx_table_match_type match_type;
3782
3783                 match_type = table_match_type_resolve(params->fields,
3784                                                       params->n_fields);
3785                 type = table_type_resolve(p,
3786                                           recommended_table_type_name,
3787                                           match_type);
3788                 CHECK(type, EINVAL);
3789         } else {
3790                 type = NULL;
3791         }
3792
3793         /* Memory allocation. */
3794         t = calloc(1, sizeof(struct table));
3795         CHECK(t, ENOMEM);
3796
3797         t->fields = calloc(params->n_fields, sizeof(struct match_field));
3798         if (!t->fields) {
3799                 free(t);
3800                 CHECK(0, ENOMEM);
3801         }
3802
3803         t->actions = calloc(params->n_actions, sizeof(struct action *));
3804         if (!t->actions) {
3805                 free(t->fields);
3806                 free(t);
3807                 CHECK(0, ENOMEM);
3808         }
3809
3810         if (action_data_size_max) {
3811                 t->default_action_data = calloc(1, action_data_size_max);
3812                 if (!t->default_action_data) {
3813                         free(t->actions);
3814                         free(t->fields);
3815                         free(t);
3816                         CHECK(0, ENOMEM);
3817                 }
3818         }
3819
3820         /* Node initialization. */
3821         strcpy(t->name, name);
3822         if (args && args[0])
3823                 strcpy(t->args, args);
3824         t->type = type;
3825
3826         for (i = 0; i < params->n_fields; i++) {
3827                 struct rte_swx_match_field_params *field = &params->fields[i];
3828                 struct match_field *f = &t->fields[i];
3829
3830                 f->match_type = field->match_type;
3831                 f->field = is_header ?
3832                         header_field_parse(p, field->name, NULL) :
3833                         metadata_field_parse(p, field->name);
3834         }
3835         t->n_fields = params->n_fields;
3836         t->is_header = is_header;
3837         t->header = header;
3838
3839         for (i = 0; i < params->n_actions; i++)
3840                 t->actions[i] = action_find(p, params->action_names[i]);
3841         t->default_action = default_action;
3842         if (default_action->st)
3843                 memcpy(t->default_action_data,
3844                        params->default_action_data,
3845                        default_action->st->n_bits / 8);
3846         t->n_actions = params->n_actions;
3847         t->default_action_is_const = params->default_action_is_const;
3848         t->action_data_size_max = action_data_size_max;
3849
3850         t->size = size;
3851         t->id = p->n_tables;
3852
3853         /* Node add to tailq. */
3854         TAILQ_INSERT_TAIL(&p->tables, t, node);
3855         p->n_tables++;
3856
3857         return 0;
3858 }
3859
3860 static struct rte_swx_table_params *
3861 table_params_get(struct table *table)
3862 {
3863         struct rte_swx_table_params *params;
3864         struct field *first, *last;
3865         uint8_t *key_mask;
3866         uint32_t key_size, key_offset, action_data_size, i;
3867
3868         /* Memory allocation. */
3869         params = calloc(1, sizeof(struct rte_swx_table_params));
3870         if (!params)
3871                 return NULL;
3872
3873         /* Key offset and size. */
3874         first = table->fields[0].field;
3875         last = table->fields[table->n_fields - 1].field;
3876         key_offset = first->offset / 8;
3877         key_size = (last->offset + last->n_bits - first->offset) / 8;
3878
3879         /* Memory allocation. */
3880         key_mask = calloc(1, key_size);
3881         if (!key_mask) {
3882                 free(params);
3883                 return NULL;
3884         }
3885
3886         /* Key mask. */
3887         for (i = 0; i < table->n_fields; i++) {
3888                 struct field *f = table->fields[i].field;
3889                 uint32_t start = (f->offset - first->offset) / 8;
3890                 size_t size = f->n_bits / 8;
3891
3892                 memset(&key_mask[start], 0xFF, size);
3893         }
3894
3895         /* Action data size. */
3896         action_data_size = 0;
3897         for (i = 0; i < table->n_actions; i++) {
3898                 struct action *action = table->actions[i];
3899                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
3900
3901                 if (ads > action_data_size)
3902                         action_data_size = ads;
3903         }
3904
3905         /* Fill in. */
3906         params->match_type = table->type->match_type;
3907         params->key_size = key_size;
3908         params->key_offset = key_offset;
3909         params->key_mask0 = key_mask;
3910         params->action_data_size = action_data_size;
3911         params->n_keys_max = table->size;
3912
3913         return params;
3914 }
3915
3916 static void
3917 table_params_free(struct rte_swx_table_params *params)
3918 {
3919         if (!params)
3920                 return;
3921
3922         free(params->key_mask0);
3923         free(params);
3924 }
3925
3926 static int
3927 table_state_build(struct rte_swx_pipeline *p)
3928 {
3929         struct table *table;
3930
3931         p->table_state = calloc(p->n_tables,
3932                                 sizeof(struct rte_swx_table_state));
3933         CHECK(p->table_state, ENOMEM);
3934
3935         TAILQ_FOREACH(table, &p->tables, node) {
3936                 struct rte_swx_table_state *ts = &p->table_state[table->id];
3937
3938                 if (table->type) {
3939                         struct rte_swx_table_params *params;
3940
3941                         /* ts->obj. */
3942                         params = table_params_get(table);
3943                         CHECK(params, ENOMEM);
3944
3945                         ts->obj = table->type->ops.create(params,
3946                                 NULL,
3947                                 table->args,
3948                                 p->numa_node);
3949
3950                         table_params_free(params);
3951                         CHECK(ts->obj, ENODEV);
3952                 }
3953
3954                 /* ts->default_action_data. */
3955                 if (table->action_data_size_max) {
3956                         ts->default_action_data =
3957                                 malloc(table->action_data_size_max);
3958                         CHECK(ts->default_action_data, ENOMEM);
3959
3960                         memcpy(ts->default_action_data,
3961                                table->default_action_data,
3962                                table->action_data_size_max);
3963                 }
3964
3965                 /* ts->default_action_id. */
3966                 ts->default_action_id = table->default_action->id;
3967         }
3968
3969         return 0;
3970 }
3971
3972 static void
3973 table_state_build_free(struct rte_swx_pipeline *p)
3974 {
3975         uint32_t i;
3976
3977         if (!p->table_state)
3978                 return;
3979
3980         for (i = 0; i < p->n_tables; i++) {
3981                 struct rte_swx_table_state *ts = &p->table_state[i];
3982                 struct table *table = table_find_by_id(p, i);
3983
3984                 /* ts->obj. */
3985                 if (table->type && ts->obj)
3986                         table->type->ops.free(ts->obj);
3987
3988                 /* ts->default_action_data. */
3989                 free(ts->default_action_data);
3990         }
3991
3992         free(p->table_state);
3993         p->table_state = NULL;
3994 }
3995
3996 static void
3997 table_state_free(struct rte_swx_pipeline *p)
3998 {
3999         table_state_build_free(p);
4000 }
4001
4002 static int
4003 table_stub_lkp(void *table __rte_unused,
4004                void *mailbox __rte_unused,
4005                uint8_t **key __rte_unused,
4006                uint64_t *action_id __rte_unused,
4007                uint8_t **action_data __rte_unused,
4008                int *hit)
4009 {
4010         *hit = 0;
4011         return 1; /* DONE. */
4012 }
4013
4014 static int
4015 table_build(struct rte_swx_pipeline *p)
4016 {
4017         uint32_t i;
4018
4019         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
4020                 struct thread *t = &p->threads[i];
4021                 struct table *table;
4022
4023                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
4024                 CHECK(t->tables, ENOMEM);
4025
4026                 TAILQ_FOREACH(table, &p->tables, node) {
4027                         struct table_runtime *r = &t->tables[table->id];
4028
4029                         if (table->type) {
4030                                 uint64_t size;
4031
4032                                 size = table->type->ops.mailbox_size_get();
4033
4034                                 /* r->func. */
4035                                 r->func = table->type->ops.lkp;
4036
4037                                 /* r->mailbox. */
4038                                 if (size) {
4039                                         r->mailbox = calloc(1, size);
4040                                         CHECK(r->mailbox, ENOMEM);
4041                                 }
4042
4043                                 /* r->key. */
4044                                 r->key = table->is_header ?
4045                                         &t->structs[table->header->struct_id] :
4046                                         &t->structs[p->metadata_struct_id];
4047                         } else {
4048                                 r->func = table_stub_lkp;
4049                         }
4050                 }
4051         }
4052
4053         return 0;
4054 }
4055
4056 static void
4057 table_build_free(struct rte_swx_pipeline *p)
4058 {
4059         uint32_t i;
4060
4061         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
4062                 struct thread *t = &p->threads[i];
4063                 uint32_t j;
4064
4065                 if (!t->tables)
4066                         continue;
4067
4068                 for (j = 0; j < p->n_tables; j++) {
4069                         struct table_runtime *r = &t->tables[j];
4070
4071                         free(r->mailbox);
4072                 }
4073
4074                 free(t->tables);
4075                 t->tables = NULL;
4076         }
4077 }
4078
4079 static void
4080 table_free(struct rte_swx_pipeline *p)
4081 {
4082         table_build_free(p);
4083
4084         /* Tables. */
4085         for ( ; ; ) {
4086                 struct table *elem;
4087
4088                 elem = TAILQ_FIRST(&p->tables);
4089                 if (!elem)
4090                         break;
4091
4092                 TAILQ_REMOVE(&p->tables, elem, node);
4093                 free(elem->fields);
4094                 free(elem->actions);
4095                 free(elem->default_action_data);
4096                 free(elem);
4097         }
4098
4099         /* Table types. */
4100         for ( ; ; ) {
4101                 struct table_type *elem;
4102
4103                 elem = TAILQ_FIRST(&p->table_types);
4104                 if (!elem)
4105                         break;
4106
4107                 TAILQ_REMOVE(&p->table_types, elem, node);
4108                 free(elem);
4109         }
4110 }
4111
4112 /*
4113  * Pipeline.
4114  */
4115 int
4116 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
4117 {
4118         struct rte_swx_pipeline *pipeline;
4119
4120         /* Check input parameters. */
4121         CHECK(p, EINVAL);
4122
4123         /* Memory allocation. */
4124         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
4125         CHECK(pipeline, ENOMEM);
4126
4127         /* Initialization. */
4128         TAILQ_INIT(&pipeline->struct_types);
4129         TAILQ_INIT(&pipeline->port_in_types);
4130         TAILQ_INIT(&pipeline->ports_in);
4131         TAILQ_INIT(&pipeline->port_out_types);
4132         TAILQ_INIT(&pipeline->ports_out);
4133         TAILQ_INIT(&pipeline->extern_types);
4134         TAILQ_INIT(&pipeline->extern_objs);
4135         TAILQ_INIT(&pipeline->extern_funcs);
4136         TAILQ_INIT(&pipeline->headers);
4137         TAILQ_INIT(&pipeline->actions);
4138         TAILQ_INIT(&pipeline->table_types);
4139         TAILQ_INIT(&pipeline->tables);
4140
4141         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
4142         pipeline->numa_node = numa_node;
4143
4144         *p = pipeline;
4145         return 0;
4146 }
4147
4148 void
4149 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
4150 {
4151         if (!p)
4152                 return;
4153
4154         free(p->instructions);
4155
4156         table_state_free(p);
4157         table_free(p);
4158         action_free(p);
4159         metadata_free(p);
4160         header_free(p);
4161         extern_func_free(p);
4162         extern_obj_free(p);
4163         port_out_free(p);
4164         port_in_free(p);
4165         struct_free(p);
4166
4167         free(p);
4168 }
4169
4170 int
4171 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
4172                                      const char **instructions,
4173                                      uint32_t n_instructions)
4174 {
4175         int err;
4176         uint32_t i;
4177
4178         err = instruction_config(p, NULL, instructions, n_instructions);
4179         if (err)
4180                 return err;
4181
4182         /* Thread instruction pointer reset. */
4183         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
4184                 struct thread *t = &p->threads[i];
4185
4186                 thread_ip_reset(p, t);
4187         }
4188
4189         return 0;
4190 }
4191
4192 int
4193 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
4194 {
4195         int status;
4196
4197         CHECK(p, EINVAL);
4198         CHECK(p->build_done == 0, EEXIST);
4199
4200         status = port_in_build(p);
4201         if (status)
4202                 goto error;
4203
4204         status = port_out_build(p);
4205         if (status)
4206                 goto error;
4207
4208         status = struct_build(p);
4209         if (status)
4210                 goto error;
4211
4212         status = extern_obj_build(p);
4213         if (status)
4214                 goto error;
4215
4216         status = extern_func_build(p);
4217         if (status)
4218                 goto error;
4219
4220         status = header_build(p);
4221         if (status)
4222                 goto error;
4223
4224         status = metadata_build(p);
4225         if (status)
4226                 goto error;
4227
4228         status = action_build(p);
4229         if (status)
4230                 goto error;
4231
4232         status = table_build(p);
4233         if (status)
4234                 goto error;
4235
4236         status = table_state_build(p);
4237         if (status)
4238                 goto error;
4239
4240         p->build_done = 1;
4241         return 0;
4242
4243 error:
4244         table_state_build_free(p);
4245         table_build_free(p);
4246         action_build_free(p);
4247         metadata_build_free(p);
4248         header_build_free(p);
4249         extern_func_build_free(p);
4250         extern_obj_build_free(p);
4251         port_out_build_free(p);
4252         port_in_build_free(p);
4253         struct_build_free(p);
4254
4255         return status;
4256 }
4257
4258 void
4259 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
4260 {
4261         uint32_t i;
4262
4263         for (i = 0; i < n_instructions; i++)
4264                 instr_exec(p);
4265 }
4266
4267 /*
4268  * Control.
4269  */
4270 int
4271 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
4272                                  struct rte_swx_table_state **table_state)
4273 {
4274         if (!p || !table_state || !p->build_done)
4275                 return -EINVAL;
4276
4277         *table_state = p->table_state;
4278         return 0;
4279 }
4280
4281 int
4282 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
4283                                  struct rte_swx_table_state *table_state)
4284 {
4285         if (!p || !table_state || !p->build_done)
4286                 return -EINVAL;
4287
4288         p->table_state = table_state;
4289         return 0;
4290 }