lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9 #include <arpa/inet.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_prefetch.h>
  13 #include <rte_byteorder.h>
  14
  15 #include "rte_swx_pipeline.h"
  16 #include "rte_swx_ctl.h"
  17
  18 #define CHECK(condition, err_code)                                             \
  19 do {                                                                           \
  20         if (!(condition))                                                      \
  21                 return -(err_code);                                            \
  22 } while (0)
  23
  24 #define CHECK_NAME(name, err_code)                                             \
  25         CHECK((name) && (name)[0], err_code)
  26
  27 #ifndef TRACE_LEVEL
  28 #define TRACE_LEVEL 0
  29 #endif
  30
  31 #if TRACE_LEVEL
  32 #define TRACE(...) printf(__VA_ARGS__)
  33 #else
  34 #define TRACE(...)
  35 #endif
  36
  37 #define ntoh64(x) rte_be_to_cpu_64(x)
  38 #define hton64(x) rte_cpu_to_be_64(x)
  39
  40 /*
  41  * Struct.
  42  */
  43 struct field {
  44         char name[RTE_SWX_NAME_SIZE];
  45         uint32_t n_bits;
  46         uint32_t offset;
  47 };
  48
  49 struct struct_type {
  50         TAILQ_ENTRY(struct_type) node;
  51         char name[RTE_SWX_NAME_SIZE];
  52         struct field *fields;
  53         uint32_t n_fields;
  54         uint32_t n_bits;
  55 };
  56
  57 TAILQ_HEAD(struct_type_tailq, struct_type);
  58
  59 /*
  60  * Input port.
  61  */
  62 struct port_in_type {
  63         TAILQ_ENTRY(port_in_type) node;
  64         char name[RTE_SWX_NAME_SIZE];
  65         struct rte_swx_port_in_ops ops;
  66 };
  67
  68 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  69
  70 struct port_in {
  71         TAILQ_ENTRY(port_in) node;
  72         struct port_in_type *type;
  73         void *obj;
  74         uint32_t id;
  75 };
  76
  77 TAILQ_HEAD(port_in_tailq, port_in);
  78
  79 struct port_in_runtime {
  80         rte_swx_port_in_pkt_rx_t pkt_rx;
  81         void *obj;
  82 };
  83
  84 /*
  85  * Output port.
  86  */
  87 struct port_out_type {
  88         TAILQ_ENTRY(port_out_type) node;
  89         char name[RTE_SWX_NAME_SIZE];
  90         struct rte_swx_port_out_ops ops;
  91 };
  92
  93 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  94
  95 struct port_out {
  96         TAILQ_ENTRY(port_out) node;
  97         struct port_out_type *type;
  98         void *obj;
  99         uint32_t id;
 100 };
 101
 102 TAILQ_HEAD(port_out_tailq, port_out);
 103
 104 struct port_out_runtime {
 105         rte_swx_port_out_pkt_tx_t pkt_tx;
 106         rte_swx_port_out_flush_t flush;
 107         void *obj;
 108 };
 109
 110 /*
 111  * Extern object.
 112  */
 113 struct extern_type_member_func {
 114         TAILQ_ENTRY(extern_type_member_func) node;
 115         char name[RTE_SWX_NAME_SIZE];
 116         rte_swx_extern_type_member_func_t func;
 117         uint32_t id;
 118 };
 119
 120 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 121
 122 struct extern_type {
 123         TAILQ_ENTRY(extern_type) node;
 124         char name[RTE_SWX_NAME_SIZE];
 125         struct struct_type *mailbox_struct_type;
 126         rte_swx_extern_type_constructor_t constructor;
 127         rte_swx_extern_type_destructor_t destructor;
 128         struct extern_type_member_func_tailq funcs;
 129         uint32_t n_funcs;
 130 };
 131
 132 TAILQ_HEAD(extern_type_tailq, extern_type);
 133
 134 struct extern_obj {
 135         TAILQ_ENTRY(extern_obj) node;
 136         char name[RTE_SWX_NAME_SIZE];
 137         struct extern_type *type;
 138         void *obj;
 139         uint32_t struct_id;
 140         uint32_t id;
 141 };
 142
 143 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 144
 145 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 146 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 147 #endif
 148
 149 struct extern_obj_runtime {
 150         void *obj;
 151         uint8_t *mailbox;
 152         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 153 };
 154
 155 /*
 156  * Extern function.
 157  */
 158 struct extern_func {
 159         TAILQ_ENTRY(extern_func) node;
 160         char name[RTE_SWX_NAME_SIZE];
 161         struct struct_type *mailbox_struct_type;
 162         rte_swx_extern_func_t func;
 163         uint32_t struct_id;
 164         uint32_t id;
 165 };
 166
 167 TAILQ_HEAD(extern_func_tailq, extern_func);
 168
 169 struct extern_func_runtime {
 170         uint8_t *mailbox;
 171         rte_swx_extern_func_t func;
 172 };
 173
 174 /*
 175  * Header.
 176  */
 177 struct header {
 178         TAILQ_ENTRY(header) node;
 179         char name[RTE_SWX_NAME_SIZE];
 180         struct struct_type *st;
 181         uint32_t struct_id;
 182         uint32_t id;
 183 };
 184
 185 TAILQ_HEAD(header_tailq, header);
 186
 187 struct header_runtime {
 188         uint8_t *ptr0;
 189 };
 190
 191 struct header_out_runtime {
 192         uint8_t *ptr0;
 193         uint8_t *ptr;
 194         uint32_t n_bytes;
 195 };
 196
 197 /*
 198  * Instruction.
 199  */
 200
 201 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 202  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 203  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 204  * when transferred to packet meta-data and in NBO when transferred to packet
 205  * headers.
 206  */
 207
 208 /* Notation conventions:
 209  *    -Header field: H = h.header.field (dst/src)
 210  *    -Meta-data field: M = m.field (dst/src)
 211  *    -Extern object mailbox field: E = e.field (dst/src)
 212  *    -Extern function mailbox field: F = f.field (dst/src)
 213  *    -Table action data field: T = t.field (src only)
 214  *    -Immediate value: I = 32-bit unsigned value (src only)
 215  */
 216
 217 enum instruction_type {
 218         /* rx m.port_in */
 219         INSTR_RX,
 220
 221         /* tx m.port_out */
 222         INSTR_TX,
 223
 224         /* extract h.header */
 225         INSTR_HDR_EXTRACT,
 226         INSTR_HDR_EXTRACT2,
 227         INSTR_HDR_EXTRACT3,
 228         INSTR_HDR_EXTRACT4,
 229         INSTR_HDR_EXTRACT5,
 230         INSTR_HDR_EXTRACT6,
 231         INSTR_HDR_EXTRACT7,
 232         INSTR_HDR_EXTRACT8,
 233
 234         /* emit h.header */
 235         INSTR_HDR_EMIT,
 236         INSTR_HDR_EMIT_TX,
 237         INSTR_HDR_EMIT2_TX,
 238         INSTR_HDR_EMIT3_TX,
 239         INSTR_HDR_EMIT4_TX,
 240         INSTR_HDR_EMIT5_TX,
 241         INSTR_HDR_EMIT6_TX,
 242         INSTR_HDR_EMIT7_TX,
 243         INSTR_HDR_EMIT8_TX,
 244
 245         /* validate h.header */
 246         INSTR_HDR_VALIDATE,
 247
 248         /* invalidate h.header */
 249         INSTR_HDR_INVALIDATE,
 250
 251         /* mov dst src
 252          * dst = src
 253          * dst = HMEF, src = HMEFTI
 254          */
 255         INSTR_MOV,   /* dst = MEF, src = MEFT */
 256         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 257         INSTR_MOV_I, /* dst = HMEF, src = I */
 258
 259         /* dma h.header t.field
 260          * memcpy(h.header, t.field, sizeof(h.header))
 261          */
 262         INSTR_DMA_HT,
 263         INSTR_DMA_HT2,
 264         INSTR_DMA_HT3,
 265         INSTR_DMA_HT4,
 266         INSTR_DMA_HT5,
 267         INSTR_DMA_HT6,
 268         INSTR_DMA_HT7,
 269         INSTR_DMA_HT8,
 270
 271         /* add dst src
 272          * dst += src
 273          * dst = HMEF, src = HMEFTI
 274          */
 275         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 276         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 277         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 278         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 279         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 280         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 281
 282         /* sub dst src
 283          * dst -= src
 284          * dst = HMEF, src = HMEFTI
 285          */
 286         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 287         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 288         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 289         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 290         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 291         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 292
 293         /* ckadd dst src
 294          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 295          * dst = H, src = {H, h.header}
 296          */
 297         INSTR_ALU_CKADD_FIELD,    /* src = H */
 298         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 299         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 300
 301         /* cksub dst src
 302          * dst = dst '- src
 303          * dst = H, src = H
 304          */
 305         INSTR_ALU_CKSUB_FIELD,
 306
 307         /* and dst src
 308          * dst &= src
 309          * dst = HMEF, src = HMEFTI
 310          */
 311         INSTR_ALU_AND,   /* dst = MEF, src = MEFT */
 312         INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 313         INSTR_ALU_AND_I, /* dst = HMEF, src = I */
 314
 315         /* or dst src
 316          * dst |= src
 317          * dst = HMEF, src = HMEFTI
 318          */
 319         INSTR_ALU_OR,   /* dst = MEF, src = MEFT */
 320         INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 321         INSTR_ALU_OR_I, /* dst = HMEF, src = I */
 322 };
 323
 324 struct instr_operand {
 325         uint8_t struct_id;
 326         uint8_t n_bits;
 327         uint8_t offset;
 328         uint8_t pad;
 329 };
 330
 331 struct instr_io {
 332         struct {
 333                 uint8_t offset;
 334                 uint8_t n_bits;
 335                 uint8_t pad[2];
 336         } io;
 337
 338         struct {
 339                 uint8_t header_id[8];
 340                 uint8_t struct_id[8];
 341                 uint8_t n_bytes[8];
 342         } hdr;
 343 };
 344
 345 struct instr_hdr_validity {
 346         uint8_t header_id;
 347 };
 348
 349 struct instr_dst_src {
 350         struct instr_operand dst;
 351         union {
 352                 struct instr_operand src;
 353                 uint32_t src_val;
 354         };
 355 };
 356
 357 struct instr_dma {
 358         struct {
 359                 uint8_t header_id[8];
 360                 uint8_t struct_id[8];
 361         } dst;
 362
 363         struct {
 364                 uint8_t offset[8];
 365         } src;
 366
 367         uint16_t n_bytes[8];
 368 };
 369
 370 struct instruction {
 371         enum instruction_type type;
 372         union {
 373                 struct instr_io io;
 374                 struct instr_hdr_validity valid;
 375                 struct instr_dst_src mov;
 376                 struct instr_dma dma;
 377                 struct instr_dst_src alu;
 378         };
 379 };
 380
 381 struct instruction_data {
 382         char label[RTE_SWX_NAME_SIZE];
 383         char jmp_label[RTE_SWX_NAME_SIZE];
 384         uint32_t n_users; /* user = jmp instruction to this instruction. */
 385         int invalid;
 386 };
 387
 388 /*
 389  * Action.
 390  */
 391 struct action {
 392         TAILQ_ENTRY(action) node;
 393         char name[RTE_SWX_NAME_SIZE];
 394         struct struct_type *st;
 395         struct instruction *instructions;
 396         uint32_t n_instructions;
 397         uint32_t id;
 398 };
 399
 400 TAILQ_HEAD(action_tailq, action);
 401
 402 /*
 403  * Table.
 404  */
 405 struct table_type {
 406         TAILQ_ENTRY(table_type) node;
 407         char name[RTE_SWX_NAME_SIZE];
 408         enum rte_swx_table_match_type match_type;
 409         struct rte_swx_table_ops ops;
 410 };
 411
 412 TAILQ_HEAD(table_type_tailq, table_type);
 413
 414 struct match_field {
 415         enum rte_swx_table_match_type match_type;
 416         struct field *field;
 417 };
 418
 419 struct table {
 420         TAILQ_ENTRY(table) node;
 421         char name[RTE_SWX_NAME_SIZE];
 422         char args[RTE_SWX_NAME_SIZE];
 423         struct table_type *type; /* NULL when n_fields == 0. */
 424
 425         /* Match. */
 426         struct match_field *fields;
 427         uint32_t n_fields;
 428         int is_header; /* Only valid when n_fields > 0. */
 429         struct header *header; /* Only valid when n_fields > 0. */
 430
 431         /* Action. */
 432         struct action **actions;
 433         struct action *default_action;
 434         uint8_t *default_action_data;
 435         uint32_t n_actions;
 436         int default_action_is_const;
 437         uint32_t action_data_size_max;
 438
 439         uint32_t size;
 440         uint32_t id;
 441 };
 442
 443 TAILQ_HEAD(table_tailq, table);
 444
 445 struct table_runtime {
 446         rte_swx_table_lookup_t func;
 447         void *mailbox;
 448         uint8_t **key;
 449 };
 450
 451 /*
 452  * Pipeline.
 453  */
 454 struct thread {
 455         /* Packet. */
 456         struct rte_swx_pkt pkt;
 457         uint8_t *ptr;
 458
 459         /* Structures. */
 460         uint8_t **structs;
 461
 462         /* Packet headers. */
 463         struct header_runtime *headers; /* Extracted or generated headers. */
 464         struct header_out_runtime *headers_out; /* Emitted headers. */
 465         uint8_t *header_storage;
 466         uint8_t *header_out_storage;
 467         uint64_t valid_headers;
 468         uint32_t n_headers_out;
 469
 470         /* Packet meta-data. */
 471         uint8_t *metadata;
 472
 473         /* Tables. */
 474         struct table_runtime *tables;
 475         struct rte_swx_table_state *table_state;
 476         uint64_t action_id;
 477         int hit; /* 0 = Miss, 1 = Hit. */
 478
 479         /* Extern objects and functions. */
 480         struct extern_obj_runtime *extern_objs;
 481         struct extern_func_runtime *extern_funcs;
 482
 483         /* Instructions. */
 484         struct instruction *ip;
 485         struct instruction *ret;
 486 };
 487
 488 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 489 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 490 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 491
 492 #define ALU(thread, ip, operator)  \
 493 {                                                                              \
 494         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 495         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 496         uint64_t dst64 = *dst64_ptr;                                           \
 497         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 498         uint64_t dst = dst64 & dst64_mask;                                     \
 499                                                                                \
 500         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 501         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 502         uint64_t src64 = *src64_ptr;                                           \
 503         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 504         uint64_t src = src64 & src64_mask;                                     \
 505                                                                                \
 506         uint64_t result = dst operator src;                                    \
 507                                                                                \
 508         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 509 }
 510
 511 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 512
 513 #define ALU_S(thread, ip, operator)  \
 514 {                                                                              \
 515         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 516         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 517         uint64_t dst64 = *dst64_ptr;                                           \
 518         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 519         uint64_t dst = dst64 & dst64_mask;                                     \
 520                                                                                \
 521         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 522         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 523         uint64_t src64 = *src64_ptr;                                           \
 524         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 525                                                                                \
 526         uint64_t result = dst operator src;                                    \
 527                                                                                \
 528         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 529 }
 530
 531 #define ALU_MH ALU_S
 532
 533 #define ALU_HM(thread, ip, operator)  \
 534 {                                                                              \
 535         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 536         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 537         uint64_t dst64 = *dst64_ptr;                                           \
 538         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 539         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 540                                                                                \
 541         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 542         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 543         uint64_t src64 = *src64_ptr;                                           \
 544         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 545         uint64_t src = src64 & src64_mask;                                     \
 546                                                                                \
 547         uint64_t result = dst operator src;                                    \
 548         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 549                                                                                \
 550         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 551 }
 552
 553 #define ALU_HH(thread, ip, operator)  \
 554 {                                                                              \
 555         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 556         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 557         uint64_t dst64 = *dst64_ptr;                                           \
 558         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 559         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 560                                                                                \
 561         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 562         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 563         uint64_t src64 = *src64_ptr;                                           \
 564         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 565                                                                                \
 566         uint64_t result = dst operator src;                                    \
 567         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 568                                                                                \
 569         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 570 }
 571
 572 #else
 573
 574 #define ALU_S ALU
 575 #define ALU_MH ALU
 576 #define ALU_HM ALU
 577 #define ALU_HH ALU
 578
 579 #endif
 580
 581 #define ALU_I(thread, ip, operator)  \
 582 {                                                                              \
 583         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 584         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 585         uint64_t dst64 = *dst64_ptr;                                           \
 586         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 587         uint64_t dst = dst64 & dst64_mask;                                     \
 588                                                                                \
 589         uint64_t src = (ip)->alu.src_val;                                      \
 590                                                                                \
 591         uint64_t result = dst operator src;                                    \
 592                                                                                \
 593         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 594 }
 595
 596 #define ALU_MI ALU_I
 597
 598 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 599
 600 #define ALU_HI(thread, ip, operator)  \
 601 {                                                                              \
 602         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 603         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 604         uint64_t dst64 = *dst64_ptr;                                           \
 605         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 606         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 607                                                                                \
 608         uint64_t src = (ip)->alu.src_val;                                      \
 609                                                                                \
 610         uint64_t result = dst operator src;                                    \
 611         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 612                                                                                \
 613         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 614 }
 615
 616 #else
 617
 618 #define ALU_HI ALU_I
 619
 620 #endif
 621
 622 #define MOV(thread, ip)  \
 623 {                                                                              \
 624         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 625         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 626         uint64_t dst64 = *dst64_ptr;                                           \
 627         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 628                                                                                \
 629         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 630         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 631         uint64_t src64 = *src64_ptr;                                           \
 632         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
 633         uint64_t src = src64 & src64_mask;                                     \
 634                                                                                \
 635         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 636 }
 637
 638 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 639
 640 #define MOV_S(thread, ip)  \
 641 {                                                                              \
 642         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 643         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 644         uint64_t dst64 = *dst64_ptr;                                           \
 645         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 646                                                                                \
 647         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 648         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 649         uint64_t src64 = *src64_ptr;                                           \
 650         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
 651                                                                                \
 652         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 653 }
 654
 655 #else
 656
 657 #define MOV_S MOV
 658
 659 #endif
 660
 661 #define MOV_I(thread, ip)  \
 662 {                                                                              \
 663         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 664         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 665         uint64_t dst64 = *dst64_ptr;                                           \
 666         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 667                                                                                \
 668         uint64_t src = (ip)->mov.src_val;                                      \
 669                                                                                \
 670         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 671 }
 672
 673 #define METADATA_READ(thread, offset, n_bits)                                  \
 674 ({                                                                             \
 675         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 676         uint64_t m64 = *m64_ptr;                                               \
 677         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 678         (m64 & m64_mask);                                                      \
 679 })
 680
 681 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
 682 {                                                                              \
 683         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 684         uint64_t m64 = *m64_ptr;                                               \
 685         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 686                                                                                \
 687         uint64_t m_new = value;                                                \
 688                                                                                \
 689         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
 690 }
 691
 692 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 693 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 694 #endif
 695
 696 struct rte_swx_pipeline {
 697         struct struct_type_tailq struct_types;
 698         struct port_in_type_tailq port_in_types;
 699         struct port_in_tailq ports_in;
 700         struct port_out_type_tailq port_out_types;
 701         struct port_out_tailq ports_out;
 702         struct extern_type_tailq extern_types;
 703         struct extern_obj_tailq extern_objs;
 704         struct extern_func_tailq extern_funcs;
 705         struct header_tailq headers;
 706         struct struct_type *metadata_st;
 707         uint32_t metadata_struct_id;
 708         struct action_tailq actions;
 709         struct table_type_tailq table_types;
 710         struct table_tailq tables;
 711
 712         struct port_in_runtime *in;
 713         struct port_out_runtime *out;
 714         struct instruction **action_instructions;
 715         struct rte_swx_table_state *table_state;
 716         struct instruction *instructions;
 717         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 718
 719         uint32_t n_structs;
 720         uint32_t n_ports_in;
 721         uint32_t n_ports_out;
 722         uint32_t n_extern_objs;
 723         uint32_t n_extern_funcs;
 724         uint32_t n_actions;
 725         uint32_t n_tables;
 726         uint32_t n_headers;
 727         uint32_t thread_id;
 728         uint32_t port_id;
 729         uint32_t n_instructions;
 730         int build_done;
 731         int numa_node;
 732 };
 733
 734 /*
 735  * Struct.
 736  */
 737 static struct struct_type *
 738 struct_type_find(struct rte_swx_pipeline *p, const char *name)
 739 {
 740         struct struct_type *elem;
 741
 742         TAILQ_FOREACH(elem, &p->struct_types, node)
 743                 if (strcmp(elem->name, name) == 0)
 744                         return elem;
 745
 746         return NULL;
 747 }
 748
 749 static struct field *
 750 struct_type_field_find(struct struct_type *st, const char *name)
 751 {
 752         uint32_t i;
 753
 754         for (i = 0; i < st->n_fields; i++) {
 755                 struct field *f = &st->fields[i];
 756
 757                 if (strcmp(f->name, name) == 0)
 758                         return f;
 759         }
 760
 761         return NULL;
 762 }
 763
 764 int
 765 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 766                                       const char *name,
 767                                       struct rte_swx_field_params *fields,
 768                                       uint32_t n_fields)
 769 {
 770         struct struct_type *st;
 771         uint32_t i;
 772
 773         CHECK(p, EINVAL);
 774         CHECK_NAME(name, EINVAL);
 775         CHECK(fields, EINVAL);
 776         CHECK(n_fields, EINVAL);
 777
 778         for (i = 0; i < n_fields; i++) {
 779                 struct rte_swx_field_params *f = &fields[i];
 780                 uint32_t j;
 781
 782                 CHECK_NAME(f->name, EINVAL);
 783                 CHECK(f->n_bits, EINVAL);
 784                 CHECK(f->n_bits <= 64, EINVAL);
 785                 CHECK((f->n_bits & 7) == 0, EINVAL);
 786
 787                 for (j = 0; j < i; j++) {
 788                         struct rte_swx_field_params *f_prev = &fields[j];
 789
 790                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 791                 }
 792         }
 793
 794         CHECK(!struct_type_find(p, name), EEXIST);
 795
 796         /* Node allocation. */
 797         st = calloc(1, sizeof(struct struct_type));
 798         CHECK(st, ENOMEM);
 799
 800         st->fields = calloc(n_fields, sizeof(struct field));
 801         if (!st->fields) {
 802                 free(st);
 803                 CHECK(0, ENOMEM);
 804         }
 805
 806         /* Node initialization. */
 807         strcpy(st->name, name);
 808         for (i = 0; i < n_fields; i++) {
 809                 struct field *dst = &st->fields[i];
 810                 struct rte_swx_field_params *src = &fields[i];
 811
 812                 strcpy(dst->name, src->name);
 813                 dst->n_bits = src->n_bits;
 814                 dst->offset = st->n_bits;
 815
 816                 st->n_bits += src->n_bits;
 817         }
 818         st->n_fields = n_fields;
 819
 820         /* Node add to tailq. */
 821         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 822
 823         return 0;
 824 }
 825
 826 static int
 827 struct_build(struct rte_swx_pipeline *p)
 828 {
 829         uint32_t i;
 830
 831         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 832                 struct thread *t = &p->threads[i];
 833
 834                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 835                 CHECK(t->structs, ENOMEM);
 836         }
 837
 838         return 0;
 839 }
 840
 841 static void
 842 struct_build_free(struct rte_swx_pipeline *p)
 843 {
 844         uint32_t i;
 845
 846         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 847                 struct thread *t = &p->threads[i];
 848
 849                 free(t->structs);
 850                 t->structs = NULL;
 851         }
 852 }
 853
 854 static void
 855 struct_free(struct rte_swx_pipeline *p)
 856 {
 857         struct_build_free(p);
 858
 859         /* Struct types. */
 860         for ( ; ; ) {
 861                 struct struct_type *elem;
 862
 863                 elem = TAILQ_FIRST(&p->struct_types);
 864                 if (!elem)
 865                         break;
 866
 867                 TAILQ_REMOVE(&p->struct_types, elem, node);
 868                 free(elem->fields);
 869                 free(elem);
 870         }
 871 }
 872
 873 /*
 874  * Input port.
 875  */
 876 static struct port_in_type *
 877 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 878 {
 879         struct port_in_type *elem;
 880
 881         if (!name)
 882                 return NULL;
 883
 884         TAILQ_FOREACH(elem, &p->port_in_types, node)
 885                 if (strcmp(elem->name, name) == 0)
 886                         return elem;
 887
 888         return NULL;
 889 }
 890
 891 int
 892 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 893                                        const char *name,
 894                                        struct rte_swx_port_in_ops *ops)
 895 {
 896         struct port_in_type *elem;
 897
 898         CHECK(p, EINVAL);
 899         CHECK_NAME(name, EINVAL);
 900         CHECK(ops, EINVAL);
 901         CHECK(ops->create, EINVAL);
 902         CHECK(ops->free, EINVAL);
 903         CHECK(ops->pkt_rx, EINVAL);
 904         CHECK(ops->stats_read, EINVAL);
 905
 906         CHECK(!port_in_type_find(p, name), EEXIST);
 907
 908         /* Node allocation. */
 909         elem = calloc(1, sizeof(struct port_in_type));
 910         CHECK(elem, ENOMEM);
 911
 912         /* Node initialization. */
 913         strcpy(elem->name, name);
 914         memcpy(&elem->ops, ops, sizeof(*ops));
 915
 916         /* Node add to tailq. */
 917         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 918
 919         return 0;
 920 }
 921
 922 static struct port_in *
 923 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 924 {
 925         struct port_in *port;
 926
 927         TAILQ_FOREACH(port, &p->ports_in, node)
 928                 if (port->id == port_id)
 929                         return port;
 930
 931         return NULL;
 932 }
 933
 934 int
 935 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 936                                 uint32_t port_id,
 937                                 const char *port_type_name,
 938                                 void *args)
 939 {
 940         struct port_in_type *type = NULL;
 941         struct port_in *port = NULL;
 942         void *obj = NULL;
 943
 944         CHECK(p, EINVAL);
 945
 946         CHECK(!port_in_find(p, port_id), EINVAL);
 947
 948         CHECK_NAME(port_type_name, EINVAL);
 949         type = port_in_type_find(p, port_type_name);
 950         CHECK(type, EINVAL);
 951
 952         obj = type->ops.create(args);
 953         CHECK(obj, ENODEV);
 954
 955         /* Node allocation. */
 956         port = calloc(1, sizeof(struct port_in));
 957         CHECK(port, ENOMEM);
 958
 959         /* Node initialization. */
 960         port->type = type;
 961         port->obj = obj;
 962         port->id = port_id;
 963
 964         /* Node add to tailq. */
 965         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 966         if (p->n_ports_in < port_id + 1)
 967                 p->n_ports_in = port_id + 1;
 968
 969         return 0;
 970 }
 971
 972 static int
 973 port_in_build(struct rte_swx_pipeline *p)
 974 {
 975         struct port_in *port;
 976         uint32_t i;
 977
 978         CHECK(p->n_ports_in, EINVAL);
 979         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 980
 981         for (i = 0; i < p->n_ports_in; i++)
 982                 CHECK(port_in_find(p, i), EINVAL);
 983
 984         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 985         CHECK(p->in, ENOMEM);
 986
 987         TAILQ_FOREACH(port, &p->ports_in, node) {
 988                 struct port_in_runtime *in = &p->in[port->id];
 989
 990                 in->pkt_rx = port->type->ops.pkt_rx;
 991                 in->obj = port->obj;
 992         }
 993
 994         return 0;
 995 }
 996
 997 static void
 998 port_in_build_free(struct rte_swx_pipeline *p)
 999 {
1000         free(p->in);
1001         p->in = NULL;
1002 }
1003
1004 static void
1005 port_in_free(struct rte_swx_pipeline *p)
1006 {
1007         port_in_build_free(p);
1008
1009         /* Input ports. */
1010         for ( ; ; ) {
1011                 struct port_in *port;
1012
1013                 port = TAILQ_FIRST(&p->ports_in);
1014                 if (!port)
1015                         break;
1016
1017                 TAILQ_REMOVE(&p->ports_in, port, node);
1018                 port->type->ops.free(port->obj);
1019                 free(port);
1020         }
1021
1022         /* Input port types. */
1023         for ( ; ; ) {
1024                 struct port_in_type *elem;
1025
1026                 elem = TAILQ_FIRST(&p->port_in_types);
1027                 if (!elem)
1028                         break;
1029
1030                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1031                 free(elem);
1032         }
1033 }
1034
1035 /*
1036  * Output port.
1037  */
1038 static struct port_out_type *
1039 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1040 {
1041         struct port_out_type *elem;
1042
1043         if (!name)
1044                 return NULL;
1045
1046         TAILQ_FOREACH(elem, &p->port_out_types, node)
1047                 if (!strcmp(elem->name, name))
1048                         return elem;
1049
1050         return NULL;
1051 }
1052
1053 int
1054 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1055                                         const char *name,
1056                                         struct rte_swx_port_out_ops *ops)
1057 {
1058         struct port_out_type *elem;
1059
1060         CHECK(p, EINVAL);
1061         CHECK_NAME(name, EINVAL);
1062         CHECK(ops, EINVAL);
1063         CHECK(ops->create, EINVAL);
1064         CHECK(ops->free, EINVAL);
1065         CHECK(ops->pkt_tx, EINVAL);
1066         CHECK(ops->stats_read, EINVAL);
1067
1068         CHECK(!port_out_type_find(p, name), EEXIST);
1069
1070         /* Node allocation. */
1071         elem = calloc(1, sizeof(struct port_out_type));
1072         CHECK(elem, ENOMEM);
1073
1074         /* Node initialization. */
1075         strcpy(elem->name, name);
1076         memcpy(&elem->ops, ops, sizeof(*ops));
1077
1078         /* Node add to tailq. */
1079         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1080
1081         return 0;
1082 }
1083
1084 static struct port_out *
1085 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1086 {
1087         struct port_out *port;
1088
1089         TAILQ_FOREACH(port, &p->ports_out, node)
1090                 if (port->id == port_id)
1091                         return port;
1092
1093         return NULL;
1094 }
1095
1096 int
1097 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1098                                  uint32_t port_id,
1099                                  const char *port_type_name,
1100                                  void *args)
1101 {
1102         struct port_out_type *type = NULL;
1103         struct port_out *port = NULL;
1104         void *obj = NULL;
1105
1106         CHECK(p, EINVAL);
1107
1108         CHECK(!port_out_find(p, port_id), EINVAL);
1109
1110         CHECK_NAME(port_type_name, EINVAL);
1111         type = port_out_type_find(p, port_type_name);
1112         CHECK(type, EINVAL);
1113
1114         obj = type->ops.create(args);
1115         CHECK(obj, ENODEV);
1116
1117         /* Node allocation. */
1118         port = calloc(1, sizeof(struct port_out));
1119         CHECK(port, ENOMEM);
1120
1121         /* Node initialization. */
1122         port->type = type;
1123         port->obj = obj;
1124         port->id = port_id;
1125
1126         /* Node add to tailq. */
1127         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1128         if (p->n_ports_out < port_id + 1)
1129                 p->n_ports_out = port_id + 1;
1130
1131         return 0;
1132 }
1133
1134 static int
1135 port_out_build(struct rte_swx_pipeline *p)
1136 {
1137         struct port_out *port;
1138         uint32_t i;
1139
1140         CHECK(p->n_ports_out, EINVAL);
1141
1142         for (i = 0; i < p->n_ports_out; i++)
1143                 CHECK(port_out_find(p, i), EINVAL);
1144
1145         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1146         CHECK(p->out, ENOMEM);
1147
1148         TAILQ_FOREACH(port, &p->ports_out, node) {
1149                 struct port_out_runtime *out = &p->out[port->id];
1150
1151                 out->pkt_tx = port->type->ops.pkt_tx;
1152                 out->flush = port->type->ops.flush;
1153                 out->obj = port->obj;
1154         }
1155
1156         return 0;
1157 }
1158
1159 static void
1160 port_out_build_free(struct rte_swx_pipeline *p)
1161 {
1162         free(p->out);
1163         p->out = NULL;
1164 }
1165
1166 static void
1167 port_out_free(struct rte_swx_pipeline *p)
1168 {
1169         port_out_build_free(p);
1170
1171         /* Output ports. */
1172         for ( ; ; ) {
1173                 struct port_out *port;
1174
1175                 port = TAILQ_FIRST(&p->ports_out);
1176                 if (!port)
1177                         break;
1178
1179                 TAILQ_REMOVE(&p->ports_out, port, node);
1180                 port->type->ops.free(port->obj);
1181                 free(port);
1182         }
1183
1184         /* Output port types. */
1185         for ( ; ; ) {
1186                 struct port_out_type *elem;
1187
1188                 elem = TAILQ_FIRST(&p->port_out_types);
1189                 if (!elem)
1190                         break;
1191
1192                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1193                 free(elem);
1194         }
1195 }
1196
1197 /*
1198  * Extern object.
1199  */
1200 static struct extern_type *
1201 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1202 {
1203         struct extern_type *elem;
1204
1205         TAILQ_FOREACH(elem, &p->extern_types, node)
1206                 if (strcmp(elem->name, name) == 0)
1207                         return elem;
1208
1209         return NULL;
1210 }
1211
1212 static struct extern_type_member_func *
1213 extern_type_member_func_find(struct extern_type *type, const char *name)
1214 {
1215         struct extern_type_member_func *elem;
1216
1217         TAILQ_FOREACH(elem, &type->funcs, node)
1218                 if (strcmp(elem->name, name) == 0)
1219                         return elem;
1220
1221         return NULL;
1222 }
1223
1224 static struct extern_obj *
1225 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1226 {
1227         struct extern_obj *elem;
1228
1229         TAILQ_FOREACH(elem, &p->extern_objs, node)
1230                 if (strcmp(elem->name, name) == 0)
1231                         return elem;
1232
1233         return NULL;
1234 }
1235
1236 static struct field *
1237 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1238                                const char *name,
1239                                struct extern_obj **object)
1240 {
1241         struct extern_obj *obj;
1242         struct field *f;
1243         char *obj_name, *field_name;
1244
1245         if ((name[0] != 'e') || (name[1] != '.'))
1246                 return NULL;
1247
1248         obj_name = strdup(&name[2]);
1249         if (!obj_name)
1250                 return NULL;
1251
1252         field_name = strchr(obj_name, '.');
1253         if (!field_name) {
1254                 free(obj_name);
1255                 return NULL;
1256         }
1257
1258         *field_name = 0;
1259         field_name++;
1260
1261         obj = extern_obj_find(p, obj_name);
1262         if (!obj) {
1263                 free(obj_name);
1264                 return NULL;
1265         }
1266
1267         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1268         if (!f) {
1269                 free(obj_name);
1270                 return NULL;
1271         }
1272
1273         if (object)
1274                 *object = obj;
1275
1276         free(obj_name);
1277         return f;
1278 }
1279
1280 int
1281 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1282         const char *name,
1283         const char *mailbox_struct_type_name,
1284         rte_swx_extern_type_constructor_t constructor,
1285         rte_swx_extern_type_destructor_t destructor)
1286 {
1287         struct extern_type *elem;
1288         struct struct_type *mailbox_struct_type;
1289
1290         CHECK(p, EINVAL);
1291
1292         CHECK_NAME(name, EINVAL);
1293         CHECK(!extern_type_find(p, name), EEXIST);
1294
1295         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1296         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1297         CHECK(mailbox_struct_type, EINVAL);
1298
1299         CHECK(constructor, EINVAL);
1300         CHECK(destructor, EINVAL);
1301
1302         /* Node allocation. */
1303         elem = calloc(1, sizeof(struct extern_type));
1304         CHECK(elem, ENOMEM);
1305
1306         /* Node initialization. */
1307         strcpy(elem->name, name);
1308         elem->mailbox_struct_type = mailbox_struct_type;
1309         elem->constructor = constructor;
1310         elem->destructor = destructor;
1311         TAILQ_INIT(&elem->funcs);
1312
1313         /* Node add to tailq. */
1314         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1315
1316         return 0;
1317 }
1318
1319 int
1320 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1321         const char *extern_type_name,
1322         const char *name,
1323         rte_swx_extern_type_member_func_t member_func)
1324 {
1325         struct extern_type *type;
1326         struct extern_type_member_func *type_member;
1327
1328         CHECK(p, EINVAL);
1329
1330         CHECK(extern_type_name, EINVAL);
1331         type = extern_type_find(p, extern_type_name);
1332         CHECK(type, EINVAL);
1333         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1334
1335         CHECK(name, EINVAL);
1336         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1337
1338         CHECK(member_func, EINVAL);
1339
1340         /* Node allocation. */
1341         type_member = calloc(1, sizeof(struct extern_type_member_func));
1342         CHECK(type_member, ENOMEM);
1343
1344         /* Node initialization. */
1345         strcpy(type_member->name, name);
1346         type_member->func = member_func;
1347         type_member->id = type->n_funcs;
1348
1349         /* Node add to tailq. */
1350         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1351         type->n_funcs++;
1352
1353         return 0;
1354 }
1355
1356 int
1357 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1358                                       const char *extern_type_name,
1359                                       const char *name,
1360                                       const char *args)
1361 {
1362         struct extern_type *type;
1363         struct extern_obj *obj;
1364         void *obj_handle;
1365
1366         CHECK(p, EINVAL);
1367
1368         CHECK_NAME(extern_type_name, EINVAL);
1369         type = extern_type_find(p, extern_type_name);
1370         CHECK(type, EINVAL);
1371
1372         CHECK_NAME(name, EINVAL);
1373         CHECK(!extern_obj_find(p, name), EEXIST);
1374
1375         /* Node allocation. */
1376         obj = calloc(1, sizeof(struct extern_obj));
1377         CHECK(obj, ENOMEM);
1378
1379         /* Object construction. */
1380         obj_handle = type->constructor(args);
1381         if (!obj_handle) {
1382                 free(obj);
1383                 CHECK(0, ENODEV);
1384         }
1385
1386         /* Node initialization. */
1387         strcpy(obj->name, name);
1388         obj->type = type;
1389         obj->obj = obj_handle;
1390         obj->struct_id = p->n_structs;
1391         obj->id = p->n_extern_objs;
1392
1393         /* Node add to tailq. */
1394         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1395         p->n_extern_objs++;
1396         p->n_structs++;
1397
1398         return 0;
1399 }
1400
1401 static int
1402 extern_obj_build(struct rte_swx_pipeline *p)
1403 {
1404         uint32_t i;
1405
1406         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1407                 struct thread *t = &p->threads[i];
1408                 struct extern_obj *obj;
1409
1410                 t->extern_objs = calloc(p->n_extern_objs,
1411                                         sizeof(struct extern_obj_runtime));
1412                 CHECK(t->extern_objs, ENOMEM);
1413
1414                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1415                         struct extern_obj_runtime *r =
1416                                 &t->extern_objs[obj->id];
1417                         struct extern_type_member_func *func;
1418                         uint32_t mailbox_size =
1419                                 obj->type->mailbox_struct_type->n_bits / 8;
1420
1421                         r->obj = obj->obj;
1422
1423                         r->mailbox = calloc(1, mailbox_size);
1424                         CHECK(r->mailbox, ENOMEM);
1425
1426                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1427                                 r->funcs[func->id] = func->func;
1428
1429                         t->structs[obj->struct_id] = r->mailbox;
1430                 }
1431         }
1432
1433         return 0;
1434 }
1435
1436 static void
1437 extern_obj_build_free(struct rte_swx_pipeline *p)
1438 {
1439         uint32_t i;
1440
1441         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1442                 struct thread *t = &p->threads[i];
1443                 uint32_t j;
1444
1445                 if (!t->extern_objs)
1446                         continue;
1447
1448                 for (j = 0; j < p->n_extern_objs; j++) {
1449                         struct extern_obj_runtime *r = &t->extern_objs[j];
1450
1451                         free(r->mailbox);
1452                 }
1453
1454                 free(t->extern_objs);
1455                 t->extern_objs = NULL;
1456         }
1457 }
1458
1459 static void
1460 extern_obj_free(struct rte_swx_pipeline *p)
1461 {
1462         extern_obj_build_free(p);
1463
1464         /* Extern objects. */
1465         for ( ; ; ) {
1466                 struct extern_obj *elem;
1467
1468                 elem = TAILQ_FIRST(&p->extern_objs);
1469                 if (!elem)
1470                         break;
1471
1472                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1473                 if (elem->obj)
1474                         elem->type->destructor(elem->obj);
1475                 free(elem);
1476         }
1477
1478         /* Extern types. */
1479         for ( ; ; ) {
1480                 struct extern_type *elem;
1481
1482                 elem = TAILQ_FIRST(&p->extern_types);
1483                 if (!elem)
1484                         break;
1485
1486                 TAILQ_REMOVE(&p->extern_types, elem, node);
1487
1488                 for ( ; ; ) {
1489                         struct extern_type_member_func *func;
1490
1491                         func = TAILQ_FIRST(&elem->funcs);
1492                         if (!func)
1493                                 break;
1494
1495                         TAILQ_REMOVE(&elem->funcs, func, node);
1496                         free(func);
1497                 }
1498
1499                 free(elem);
1500         }
1501 }
1502
1503 /*
1504  * Extern function.
1505  */
1506 static struct extern_func *
1507 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1508 {
1509         struct extern_func *elem;
1510
1511         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1512                 if (strcmp(elem->name, name) == 0)
1513                         return elem;
1514
1515         return NULL;
1516 }
1517
1518 static struct field *
1519 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1520                                 const char *name,
1521                                 struct extern_func **function)
1522 {
1523         struct extern_func *func;
1524         struct field *f;
1525         char *func_name, *field_name;
1526
1527         if ((name[0] != 'f') || (name[1] != '.'))
1528                 return NULL;
1529
1530         func_name = strdup(&name[2]);
1531         if (!func_name)
1532                 return NULL;
1533
1534         field_name = strchr(func_name, '.');
1535         if (!field_name) {
1536                 free(func_name);
1537                 return NULL;
1538         }
1539
1540         *field_name = 0;
1541         field_name++;
1542
1543         func = extern_func_find(p, func_name);
1544         if (!func) {
1545                 free(func_name);
1546                 return NULL;
1547         }
1548
1549         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1550         if (!f) {
1551                 free(func_name);
1552                 return NULL;
1553         }
1554
1555         if (function)
1556                 *function = func;
1557
1558         free(func_name);
1559         return f;
1560 }
1561
1562 int
1563 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1564                                       const char *name,
1565                                       const char *mailbox_struct_type_name,
1566                                       rte_swx_extern_func_t func)
1567 {
1568         struct extern_func *f;
1569         struct struct_type *mailbox_struct_type;
1570
1571         CHECK(p, EINVAL);
1572
1573         CHECK_NAME(name, EINVAL);
1574         CHECK(!extern_func_find(p, name), EEXIST);
1575
1576         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1577         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1578         CHECK(mailbox_struct_type, EINVAL);
1579
1580         CHECK(func, EINVAL);
1581
1582         /* Node allocation. */
1583         f = calloc(1, sizeof(struct extern_func));
1584         CHECK(func, ENOMEM);
1585
1586         /* Node initialization. */
1587         strcpy(f->name, name);
1588         f->mailbox_struct_type = mailbox_struct_type;
1589         f->func = func;
1590         f->struct_id = p->n_structs;
1591         f->id = p->n_extern_funcs;
1592
1593         /* Node add to tailq. */
1594         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1595         p->n_extern_funcs++;
1596         p->n_structs++;
1597
1598         return 0;
1599 }
1600
1601 static int
1602 extern_func_build(struct rte_swx_pipeline *p)
1603 {
1604         uint32_t i;
1605
1606         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1607                 struct thread *t = &p->threads[i];
1608                 struct extern_func *func;
1609
1610                 /* Memory allocation. */
1611                 t->extern_funcs = calloc(p->n_extern_funcs,
1612                                          sizeof(struct extern_func_runtime));
1613                 CHECK(t->extern_funcs, ENOMEM);
1614
1615                 /* Extern function. */
1616                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1617                         struct extern_func_runtime *r =
1618                                 &t->extern_funcs[func->id];
1619                         uint32_t mailbox_size =
1620                                 func->mailbox_struct_type->n_bits / 8;
1621
1622                         r->func = func->func;
1623
1624                         r->mailbox = calloc(1, mailbox_size);
1625                         CHECK(r->mailbox, ENOMEM);
1626
1627                         t->structs[func->struct_id] = r->mailbox;
1628                 }
1629         }
1630
1631         return 0;
1632 }
1633
1634 static void
1635 extern_func_build_free(struct rte_swx_pipeline *p)
1636 {
1637         uint32_t i;
1638
1639         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1640                 struct thread *t = &p->threads[i];
1641                 uint32_t j;
1642
1643                 if (!t->extern_funcs)
1644                         continue;
1645
1646                 for (j = 0; j < p->n_extern_funcs; j++) {
1647                         struct extern_func_runtime *r = &t->extern_funcs[j];
1648
1649                         free(r->mailbox);
1650                 }
1651
1652                 free(t->extern_funcs);
1653                 t->extern_funcs = NULL;
1654         }
1655 }
1656
1657 static void
1658 extern_func_free(struct rte_swx_pipeline *p)
1659 {
1660         extern_func_build_free(p);
1661
1662         for ( ; ; ) {
1663                 struct extern_func *elem;
1664
1665                 elem = TAILQ_FIRST(&p->extern_funcs);
1666                 if (!elem)
1667                         break;
1668
1669                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1670                 free(elem);
1671         }
1672 }
1673
1674 /*
1675  * Header.
1676  */
1677 static struct header *
1678 header_find(struct rte_swx_pipeline *p, const char *name)
1679 {
1680         struct header *elem;
1681
1682         TAILQ_FOREACH(elem, &p->headers, node)
1683                 if (strcmp(elem->name, name) == 0)
1684                         return elem;
1685
1686         return NULL;
1687 }
1688
1689 static struct header *
1690 header_parse(struct rte_swx_pipeline *p,
1691              const char *name)
1692 {
1693         if (name[0] != 'h' || name[1] != '.')
1694                 return NULL;
1695
1696         return header_find(p, &name[2]);
1697 }
1698
1699 static struct field *
1700 header_field_parse(struct rte_swx_pipeline *p,
1701                    const char *name,
1702                    struct header **header)
1703 {
1704         struct header *h;
1705         struct field *f;
1706         char *header_name, *field_name;
1707
1708         if ((name[0] != 'h') || (name[1] != '.'))
1709                 return NULL;
1710
1711         header_name = strdup(&name[2]);
1712         if (!header_name)
1713                 return NULL;
1714
1715         field_name = strchr(header_name, '.');
1716         if (!field_name) {
1717                 free(header_name);
1718                 return NULL;
1719         }
1720
1721         *field_name = 0;
1722         field_name++;
1723
1724         h = header_find(p, header_name);
1725         if (!h) {
1726                 free(header_name);
1727                 return NULL;
1728         }
1729
1730         f = struct_type_field_find(h->st, field_name);
1731         if (!f) {
1732                 free(header_name);
1733                 return NULL;
1734         }
1735
1736         if (header)
1737                 *header = h;
1738
1739         free(header_name);
1740         return f;
1741 }
1742
1743 int
1744 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1745                                         const char *name,
1746                                         const char *struct_type_name)
1747 {
1748         struct struct_type *st;
1749         struct header *h;
1750         size_t n_headers_max;
1751
1752         CHECK(p, EINVAL);
1753         CHECK_NAME(name, EINVAL);
1754         CHECK_NAME(struct_type_name, EINVAL);
1755
1756         CHECK(!header_find(p, name), EEXIST);
1757
1758         st = struct_type_find(p, struct_type_name);
1759         CHECK(st, EINVAL);
1760
1761         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1762         CHECK(p->n_headers < n_headers_max, ENOSPC);
1763
1764         /* Node allocation. */
1765         h = calloc(1, sizeof(struct header));
1766         CHECK(h, ENOMEM);
1767
1768         /* Node initialization. */
1769         strcpy(h->name, name);
1770         h->st = st;
1771         h->struct_id = p->n_structs;
1772         h->id = p->n_headers;
1773
1774         /* Node add to tailq. */
1775         TAILQ_INSERT_TAIL(&p->headers, h, node);
1776         p->n_headers++;
1777         p->n_structs++;
1778
1779         return 0;
1780 }
1781
1782 static int
1783 header_build(struct rte_swx_pipeline *p)
1784 {
1785         struct header *h;
1786         uint32_t n_bytes = 0, i;
1787
1788         TAILQ_FOREACH(h, &p->headers, node) {
1789                 n_bytes += h->st->n_bits / 8;
1790         }
1791
1792         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1793                 struct thread *t = &p->threads[i];
1794                 uint32_t offset = 0;
1795
1796                 t->headers = calloc(p->n_headers,
1797                                     sizeof(struct header_runtime));
1798                 CHECK(t->headers, ENOMEM);
1799
1800                 t->headers_out = calloc(p->n_headers,
1801                                         sizeof(struct header_out_runtime));
1802                 CHECK(t->headers_out, ENOMEM);
1803
1804                 t->header_storage = calloc(1, n_bytes);
1805                 CHECK(t->header_storage, ENOMEM);
1806
1807                 t->header_out_storage = calloc(1, n_bytes);
1808                 CHECK(t->header_out_storage, ENOMEM);
1809
1810                 TAILQ_FOREACH(h, &p->headers, node) {
1811                         uint8_t *header_storage;
1812
1813                         header_storage = &t->header_storage[offset];
1814                         offset += h->st->n_bits / 8;
1815
1816                         t->headers[h->id].ptr0 = header_storage;
1817                         t->structs[h->struct_id] = header_storage;
1818                 }
1819         }
1820
1821         return 0;
1822 }
1823
1824 static void
1825 header_build_free(struct rte_swx_pipeline *p)
1826 {
1827         uint32_t i;
1828
1829         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1830                 struct thread *t = &p->threads[i];
1831
1832                 free(t->headers_out);
1833                 t->headers_out = NULL;
1834
1835                 free(t->headers);
1836                 t->headers = NULL;
1837
1838                 free(t->header_out_storage);
1839                 t->header_out_storage = NULL;
1840
1841                 free(t->header_storage);
1842                 t->header_storage = NULL;
1843         }
1844 }
1845
1846 static void
1847 header_free(struct rte_swx_pipeline *p)
1848 {
1849         header_build_free(p);
1850
1851         for ( ; ; ) {
1852                 struct header *elem;
1853
1854                 elem = TAILQ_FIRST(&p->headers);
1855                 if (!elem)
1856                         break;
1857
1858                 TAILQ_REMOVE(&p->headers, elem, node);
1859                 free(elem);
1860         }
1861 }
1862
1863 /*
1864  * Meta-data.
1865  */
1866 static struct field *
1867 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1868 {
1869         if (!p->metadata_st)
1870                 return NULL;
1871
1872         if (name[0] != 'm' || name[1] != '.')
1873                 return NULL;
1874
1875         return struct_type_field_find(p->metadata_st, &name[2]);
1876 }
1877
1878 int
1879 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1880                                           const char *struct_type_name)
1881 {
1882         struct struct_type *st = NULL;
1883
1884         CHECK(p, EINVAL);
1885
1886         CHECK_NAME(struct_type_name, EINVAL);
1887         st  = struct_type_find(p, struct_type_name);
1888         CHECK(st, EINVAL);
1889         CHECK(!p->metadata_st, EINVAL);
1890
1891         p->metadata_st = st;
1892         p->metadata_struct_id = p->n_structs;
1893
1894         p->n_structs++;
1895
1896         return 0;
1897 }
1898
1899 static int
1900 metadata_build(struct rte_swx_pipeline *p)
1901 {
1902         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1903         uint32_t i;
1904
1905         /* Thread-level initialization. */
1906         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1907                 struct thread *t = &p->threads[i];
1908                 uint8_t *metadata;
1909
1910                 metadata = calloc(1, n_bytes);
1911                 CHECK(metadata, ENOMEM);
1912
1913                 t->metadata = metadata;
1914                 t->structs[p->metadata_struct_id] = metadata;
1915         }
1916
1917         return 0;
1918 }
1919
1920 static void
1921 metadata_build_free(struct rte_swx_pipeline *p)
1922 {
1923         uint32_t i;
1924
1925         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1926                 struct thread *t = &p->threads[i];
1927
1928                 free(t->metadata);
1929                 t->metadata = NULL;
1930         }
1931 }
1932
1933 static void
1934 metadata_free(struct rte_swx_pipeline *p)
1935 {
1936         metadata_build_free(p);
1937 }
1938
1939 /*
1940  * Instruction.
1941  */
1942 static struct field *
1943 action_field_parse(struct action *action, const char *name);
1944
1945 static struct field *
1946 struct_field_parse(struct rte_swx_pipeline *p,
1947                    struct action *action,
1948                    const char *name,
1949                    uint32_t *struct_id)
1950 {
1951         struct field *f;
1952
1953         switch (name[0]) {
1954         case 'h':
1955         {
1956                 struct header *header;
1957
1958                 f = header_field_parse(p, name, &header);
1959                 if (!f)
1960                         return NULL;
1961
1962                 *struct_id = header->struct_id;
1963                 return f;
1964         }
1965
1966         case 'm':
1967         {
1968                 f = metadata_field_parse(p, name);
1969                 if (!f)
1970                         return NULL;
1971
1972                 *struct_id = p->metadata_struct_id;
1973                 return f;
1974         }
1975
1976         case 't':
1977         {
1978                 if (!action)
1979                         return NULL;
1980
1981                 f = action_field_parse(action, name);
1982                 if (!f)
1983                         return NULL;
1984
1985                 *struct_id = 0;
1986                 return f;
1987         }
1988
1989         case 'e':
1990         {
1991                 struct extern_obj *obj;
1992
1993                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1994                 if (!f)
1995                         return NULL;
1996
1997                 *struct_id = obj->struct_id;
1998                 return f;
1999         }
2000
2001         case 'f':
2002         {
2003                 struct extern_func *func;
2004
2005                 f = extern_func_mailbox_field_parse(p, name, &func);
2006                 if (!f)
2007                         return NULL;
2008
2009                 *struct_id = func->struct_id;
2010                 return f;
2011         }
2012
2013         default:
2014                 return NULL;
2015         }
2016 }
2017
2018 static inline void
2019 pipeline_port_inc(struct rte_swx_pipeline *p)
2020 {
2021         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2022 }
2023
2024 static inline void
2025 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2026 {
2027         t->ip = p->instructions;
2028 }
2029
2030 static inline void
2031 thread_ip_inc(struct rte_swx_pipeline *p);
2032
2033 static inline void
2034 thread_ip_inc(struct rte_swx_pipeline *p)
2035 {
2036         struct thread *t = &p->threads[p->thread_id];
2037
2038         t->ip++;
2039 }
2040
2041 static inline void
2042 thread_ip_inc_cond(struct thread *t, int cond)
2043 {
2044         t->ip += cond;
2045 }
2046
2047 static inline void
2048 thread_yield(struct rte_swx_pipeline *p)
2049 {
2050         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2051 }
2052
2053 /*
2054  * rx.
2055  */
2056 static int
2057 instr_rx_translate(struct rte_swx_pipeline *p,
2058                    struct action *action,
2059                    char **tokens,
2060                    int n_tokens,
2061                    struct instruction *instr,
2062                    struct instruction_data *data __rte_unused)
2063 {
2064         struct field *f;
2065
2066         CHECK(!action, EINVAL);
2067         CHECK(n_tokens == 2, EINVAL);
2068
2069         f = metadata_field_parse(p, tokens[1]);
2070         CHECK(f, EINVAL);
2071
2072         instr->type = INSTR_RX;
2073         instr->io.io.offset = f->offset / 8;
2074         instr->io.io.n_bits = f->n_bits;
2075         return 0;
2076 }
2077
2078 static inline void
2079 instr_rx_exec(struct rte_swx_pipeline *p);
2080
2081 static inline void
2082 instr_rx_exec(struct rte_swx_pipeline *p)
2083 {
2084         struct thread *t = &p->threads[p->thread_id];
2085         struct instruction *ip = t->ip;
2086         struct port_in_runtime *port = &p->in[p->port_id];
2087         struct rte_swx_pkt *pkt = &t->pkt;
2088         int pkt_received;
2089
2090         /* Packet. */
2091         pkt_received = port->pkt_rx(port->obj, pkt);
2092         t->ptr = &pkt->pkt[pkt->offset];
2093         rte_prefetch0(t->ptr);
2094
2095         TRACE("[Thread %2u] rx %s from port %u\n",
2096               p->thread_id,
2097               pkt_received ? "1 pkt" : "0 pkts",
2098               p->port_id);
2099
2100         /* Headers. */
2101         t->valid_headers = 0;
2102         t->n_headers_out = 0;
2103
2104         /* Meta-data. */
2105         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2106
2107         /* Tables. */
2108         t->table_state = p->table_state;
2109
2110         /* Thread. */
2111         pipeline_port_inc(p);
2112         thread_ip_inc_cond(t, pkt_received);
2113         thread_yield(p);
2114 }
2115
2116 /*
2117  * tx.
2118  */
2119 static int
2120 instr_tx_translate(struct rte_swx_pipeline *p,
2121                    struct action *action __rte_unused,
2122                    char **tokens,
2123                    int n_tokens,
2124                    struct instruction *instr,
2125                    struct instruction_data *data __rte_unused)
2126 {
2127         struct field *f;
2128
2129         CHECK(n_tokens == 2, EINVAL);
2130
2131         f = metadata_field_parse(p, tokens[1]);
2132         CHECK(f, EINVAL);
2133
2134         instr->type = INSTR_TX;
2135         instr->io.io.offset = f->offset / 8;
2136         instr->io.io.n_bits = f->n_bits;
2137         return 0;
2138 }
2139
2140 static inline void
2141 emit_handler(struct thread *t)
2142 {
2143         struct header_out_runtime *h0 = &t->headers_out[0];
2144         struct header_out_runtime *h1 = &t->headers_out[1];
2145         uint32_t offset = 0, i;
2146
2147         /* No header change or header decapsulation. */
2148         if ((t->n_headers_out == 1) &&
2149             (h0->ptr + h0->n_bytes == t->ptr)) {
2150                 TRACE("Emit handler: no header change or header decap.\n");
2151
2152                 t->pkt.offset -= h0->n_bytes;
2153                 t->pkt.length += h0->n_bytes;
2154
2155                 return;
2156         }
2157
2158         /* Header encapsulation (optionally, with prior header decasulation). */
2159         if ((t->n_headers_out == 2) &&
2160             (h1->ptr + h1->n_bytes == t->ptr) &&
2161             (h0->ptr == h0->ptr0)) {
2162                 uint32_t offset;
2163
2164                 TRACE("Emit handler: header encapsulation.\n");
2165
2166                 offset = h0->n_bytes + h1->n_bytes;
2167                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2168                 t->pkt.offset -= offset;
2169                 t->pkt.length += offset;
2170
2171                 return;
2172         }
2173
2174         /* Header insertion. */
2175         /* TBD */
2176
2177         /* Header extraction. */
2178         /* TBD */
2179
2180         /* For any other case. */
2181         TRACE("Emit handler: complex case.\n");
2182
2183         for (i = 0; i < t->n_headers_out; i++) {
2184                 struct header_out_runtime *h = &t->headers_out[i];
2185
2186                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2187                 offset += h->n_bytes;
2188         }
2189
2190         if (offset) {
2191                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2192                 t->pkt.offset -= offset;
2193                 t->pkt.length += offset;
2194         }
2195 }
2196
2197 static inline void
2198 instr_tx_exec(struct rte_swx_pipeline *p);
2199
2200 static inline void
2201 instr_tx_exec(struct rte_swx_pipeline *p)
2202 {
2203         struct thread *t = &p->threads[p->thread_id];
2204         struct instruction *ip = t->ip;
2205         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2206         struct port_out_runtime *port = &p->out[port_id];
2207         struct rte_swx_pkt *pkt = &t->pkt;
2208
2209         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2210               p->thread_id,
2211               (uint32_t)port_id);
2212
2213         /* Headers. */
2214         emit_handler(t);
2215
2216         /* Packet. */
2217         port->pkt_tx(port->obj, pkt);
2218
2219         /* Thread. */
2220         thread_ip_reset(p, t);
2221         instr_rx_exec(p);
2222 }
2223
2224 /*
2225  * extract.
2226  */
2227 static int
2228 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2229                             struct action *action,
2230                             char **tokens,
2231                             int n_tokens,
2232                             struct instruction *instr,
2233                             struct instruction_data *data __rte_unused)
2234 {
2235         struct header *h;
2236
2237         CHECK(!action, EINVAL);
2238         CHECK(n_tokens == 2, EINVAL);
2239
2240         h = header_parse(p, tokens[1]);
2241         CHECK(h, EINVAL);
2242
2243         instr->type = INSTR_HDR_EXTRACT;
2244         instr->io.hdr.header_id[0] = h->id;
2245         instr->io.hdr.struct_id[0] = h->struct_id;
2246         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2247         return 0;
2248 }
2249
2250 static inline void
2251 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2252
2253 static inline void
2254 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2255 {
2256         struct thread *t = &p->threads[p->thread_id];
2257         struct instruction *ip = t->ip;
2258         uint64_t valid_headers = t->valid_headers;
2259         uint8_t *ptr = t->ptr;
2260         uint32_t offset = t->pkt.offset;
2261         uint32_t length = t->pkt.length;
2262         uint32_t i;
2263
2264         for (i = 0; i < n_extract; i++) {
2265                 uint32_t header_id = ip->io.hdr.header_id[i];
2266                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2267                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2268
2269                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2270                       p->thread_id,
2271                       header_id,
2272                       n_bytes);
2273
2274                 /* Headers. */
2275                 t->structs[struct_id] = ptr;
2276                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2277
2278                 /* Packet. */
2279                 offset += n_bytes;
2280                 length -= n_bytes;
2281                 ptr += n_bytes;
2282         }
2283
2284         /* Headers. */
2285         t->valid_headers = valid_headers;
2286
2287         /* Packet. */
2288         t->pkt.offset = offset;
2289         t->pkt.length = length;
2290         t->ptr = ptr;
2291 }
2292
2293 static inline void
2294 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2295 {
2296         __instr_hdr_extract_exec(p, 1);
2297
2298         /* Thread. */
2299         thread_ip_inc(p);
2300 }
2301
2302 static inline void
2303 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2304 {
2305         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2306               p->thread_id);
2307
2308         __instr_hdr_extract_exec(p, 2);
2309
2310         /* Thread. */
2311         thread_ip_inc(p);
2312 }
2313
2314 static inline void
2315 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2316 {
2317         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2318               p->thread_id);
2319
2320         __instr_hdr_extract_exec(p, 3);
2321
2322         /* Thread. */
2323         thread_ip_inc(p);
2324 }
2325
2326 static inline void
2327 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2328 {
2329         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2330               p->thread_id);
2331
2332         __instr_hdr_extract_exec(p, 4);
2333
2334         /* Thread. */
2335         thread_ip_inc(p);
2336 }
2337
2338 static inline void
2339 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2340 {
2341         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2342               p->thread_id);
2343
2344         __instr_hdr_extract_exec(p, 5);
2345
2346         /* Thread. */
2347         thread_ip_inc(p);
2348 }
2349
2350 static inline void
2351 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2352 {
2353         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2354               p->thread_id);
2355
2356         __instr_hdr_extract_exec(p, 6);
2357
2358         /* Thread. */
2359         thread_ip_inc(p);
2360 }
2361
2362 static inline void
2363 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2364 {
2365         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2366               p->thread_id);
2367
2368         __instr_hdr_extract_exec(p, 7);
2369
2370         /* Thread. */
2371         thread_ip_inc(p);
2372 }
2373
2374 static inline void
2375 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2376 {
2377         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2378               p->thread_id);
2379
2380         __instr_hdr_extract_exec(p, 8);
2381
2382         /* Thread. */
2383         thread_ip_inc(p);
2384 }
2385
2386 /*
2387  * emit.
2388  */
2389 static int
2390 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2391                          struct action *action __rte_unused,
2392                          char **tokens,
2393                          int n_tokens,
2394                          struct instruction *instr,
2395                          struct instruction_data *data __rte_unused)
2396 {
2397         struct header *h;
2398
2399         CHECK(n_tokens == 2, EINVAL);
2400
2401         h = header_parse(p, tokens[1]);
2402         CHECK(h, EINVAL);
2403
2404         instr->type = INSTR_HDR_EMIT;
2405         instr->io.hdr.header_id[0] = h->id;
2406         instr->io.hdr.struct_id[0] = h->struct_id;
2407         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2408         return 0;
2409 }
2410
2411 static inline void
2412 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
2413
2414 static inline void
2415 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
2416 {
2417         struct thread *t = &p->threads[p->thread_id];
2418         struct instruction *ip = t->ip;
2419         uint32_t n_headers_out = t->n_headers_out;
2420         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
2421         uint8_t *ho_ptr = NULL;
2422         uint32_t ho_nbytes = 0, i;
2423
2424         for (i = 0; i < n_emit; i++) {
2425                 uint32_t header_id = ip->io.hdr.header_id[i];
2426                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2427                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2428
2429                 struct header_runtime *hi = &t->headers[header_id];
2430                 uint8_t *hi_ptr = t->structs[struct_id];
2431
2432                 TRACE("[Thread %2u]: emit header %u\n",
2433                       p->thread_id,
2434                       header_id);
2435
2436                 /* Headers. */
2437                 if (!i) {
2438                         if (!t->n_headers_out) {
2439                                 ho = &t->headers_out[0];
2440
2441                                 ho->ptr0 = hi->ptr0;
2442                                 ho->ptr = hi_ptr;
2443
2444                                 ho_ptr = hi_ptr;
2445                                 ho_nbytes = n_bytes;
2446
2447                                 n_headers_out = 1;
2448
2449                                 continue;
2450                         } else {
2451                                 ho_ptr = ho->ptr;
2452                                 ho_nbytes = ho->n_bytes;
2453                         }
2454                 }
2455
2456                 if (ho_ptr + ho_nbytes == hi_ptr) {
2457                         ho_nbytes += n_bytes;
2458                 } else {
2459                         ho->n_bytes = ho_nbytes;
2460
2461                         ho++;
2462                         ho->ptr0 = hi->ptr0;
2463                         ho->ptr = hi_ptr;
2464
2465                         ho_ptr = hi_ptr;
2466                         ho_nbytes = n_bytes;
2467
2468                         n_headers_out++;
2469                 }
2470         }
2471
2472         ho->n_bytes = ho_nbytes;
2473         t->n_headers_out = n_headers_out;
2474 }
2475
2476 static inline void
2477 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2478 {
2479         __instr_hdr_emit_exec(p, 1);
2480
2481         /* Thread. */
2482         thread_ip_inc(p);
2483 }
2484
2485 static inline void
2486 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2487 {
2488         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2489               p->thread_id);
2490
2491         __instr_hdr_emit_exec(p, 1);
2492         instr_tx_exec(p);
2493 }
2494
2495 static inline void
2496 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2497 {
2498         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2499               p->thread_id);
2500
2501         __instr_hdr_emit_exec(p, 2);
2502         instr_tx_exec(p);
2503 }
2504
2505 static inline void
2506 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2507 {
2508         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2509               p->thread_id);
2510
2511         __instr_hdr_emit_exec(p, 3);
2512         instr_tx_exec(p);
2513 }
2514
2515 static inline void
2516 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2517 {
2518         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2519               p->thread_id);
2520
2521         __instr_hdr_emit_exec(p, 4);
2522         instr_tx_exec(p);
2523 }
2524
2525 static inline void
2526 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2527 {
2528         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2529               p->thread_id);
2530
2531         __instr_hdr_emit_exec(p, 5);
2532         instr_tx_exec(p);
2533 }
2534
2535 static inline void
2536 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2537 {
2538         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2539               p->thread_id);
2540
2541         __instr_hdr_emit_exec(p, 6);
2542         instr_tx_exec(p);
2543 }
2544
2545 static inline void
2546 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2547 {
2548         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2549               p->thread_id);
2550
2551         __instr_hdr_emit_exec(p, 7);
2552         instr_tx_exec(p);
2553 }
2554
2555 static inline void
2556 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2557 {
2558         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2559               p->thread_id);
2560
2561         __instr_hdr_emit_exec(p, 8);
2562         instr_tx_exec(p);
2563 }
2564
2565 /*
2566  * validate.
2567  */
2568 static int
2569 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2570                              struct action *action __rte_unused,
2571                              char **tokens,
2572                              int n_tokens,
2573                              struct instruction *instr,
2574                              struct instruction_data *data __rte_unused)
2575 {
2576         struct header *h;
2577
2578         CHECK(n_tokens == 2, EINVAL);
2579
2580         h = header_parse(p, tokens[1]);
2581         CHECK(h, EINVAL);
2582
2583         instr->type = INSTR_HDR_VALIDATE;
2584         instr->valid.header_id = h->id;
2585         return 0;
2586 }
2587
2588 static inline void
2589 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2590 {
2591         struct thread *t = &p->threads[p->thread_id];
2592         struct instruction *ip = t->ip;
2593         uint32_t header_id = ip->valid.header_id;
2594
2595         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2596
2597         /* Headers. */
2598         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2599
2600         /* Thread. */
2601         thread_ip_inc(p);
2602 }
2603
2604 /*
2605  * invalidate.
2606  */
2607 static int
2608 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2609                                struct action *action __rte_unused,
2610                                char **tokens,
2611                                int n_tokens,
2612                                struct instruction *instr,
2613                                struct instruction_data *data __rte_unused)
2614 {
2615         struct header *h;
2616
2617         CHECK(n_tokens == 2, EINVAL);
2618
2619         h = header_parse(p, tokens[1]);
2620         CHECK(h, EINVAL);
2621
2622         instr->type = INSTR_HDR_INVALIDATE;
2623         instr->valid.header_id = h->id;
2624         return 0;
2625 }
2626
2627 static inline void
2628 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2629 {
2630         struct thread *t = &p->threads[p->thread_id];
2631         struct instruction *ip = t->ip;
2632         uint32_t header_id = ip->valid.header_id;
2633
2634         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2635
2636         /* Headers. */
2637         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2638
2639         /* Thread. */
2640         thread_ip_inc(p);
2641 }
2642
2643 /*
2644  * mov.
2645  */
2646 static int
2647 instr_mov_translate(struct rte_swx_pipeline *p,
2648                     struct action *action,
2649                     char **tokens,
2650                     int n_tokens,
2651                     struct instruction *instr,
2652                     struct instruction_data *data __rte_unused)
2653 {
2654         char *dst = tokens[1], *src = tokens[2];
2655         struct field *fdst, *fsrc;
2656         uint32_t dst_struct_id, src_struct_id, src_val;
2657
2658         CHECK(n_tokens == 3, EINVAL);
2659
2660         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2661         CHECK(fdst, EINVAL);
2662
2663         /* MOV or MOV_S. */
2664         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2665         if (fsrc) {
2666                 instr->type = INSTR_MOV;
2667                 if ((dst[0] == 'h' && src[0] != 'h') ||
2668                     (dst[0] != 'h' && src[0] == 'h'))
2669                         instr->type = INSTR_MOV_S;
2670
2671                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2672                 instr->mov.dst.n_bits = fdst->n_bits;
2673                 instr->mov.dst.offset = fdst->offset / 8;
2674                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2675                 instr->mov.src.n_bits = fsrc->n_bits;
2676                 instr->mov.src.offset = fsrc->offset / 8;
2677                 return 0;
2678         }
2679
2680         /* MOV_I. */
2681         src_val = strtoul(src, &src, 0);
2682         CHECK(!src[0], EINVAL);
2683
2684         if (dst[0] == 'h')
2685                 src_val = htonl(src_val);
2686
2687         instr->type = INSTR_MOV_I;
2688         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2689         instr->mov.dst.n_bits = fdst->n_bits;
2690         instr->mov.dst.offset = fdst->offset / 8;
2691         instr->mov.src_val = (uint32_t)src_val;
2692         return 0;
2693 }
2694
2695 static inline void
2696 instr_mov_exec(struct rte_swx_pipeline *p)
2697 {
2698         struct thread *t = &p->threads[p->thread_id];
2699         struct instruction *ip = t->ip;
2700
2701         TRACE("[Thread %2u] mov\n",
2702               p->thread_id);
2703
2704         MOV(t, ip);
2705
2706         /* Thread. */
2707         thread_ip_inc(p);
2708 }
2709
2710 static inline void
2711 instr_mov_s_exec(struct rte_swx_pipeline *p)
2712 {
2713         struct thread *t = &p->threads[p->thread_id];
2714         struct instruction *ip = t->ip;
2715
2716         TRACE("[Thread %2u] mov (s)\n",
2717               p->thread_id);
2718
2719         MOV_S(t, ip);
2720
2721         /* Thread. */
2722         thread_ip_inc(p);
2723 }
2724
2725 static inline void
2726 instr_mov_i_exec(struct rte_swx_pipeline *p)
2727 {
2728         struct thread *t = &p->threads[p->thread_id];
2729         struct instruction *ip = t->ip;
2730
2731         TRACE("[Thread %2u] mov m.f %x\n",
2732               p->thread_id,
2733               ip->mov.src_val);
2734
2735         MOV_I(t, ip);
2736
2737         /* Thread. */
2738         thread_ip_inc(p);
2739 }
2740
2741 /*
2742  * dma.
2743  */
2744 static int
2745 instr_dma_translate(struct rte_swx_pipeline *p,
2746                     struct action *action,
2747                     char **tokens,
2748                     int n_tokens,
2749                     struct instruction *instr,
2750                     struct instruction_data *data __rte_unused)
2751 {
2752         char *dst = tokens[1];
2753         char *src = tokens[2];
2754         struct header *h;
2755         struct field *tf;
2756
2757         CHECK(action, EINVAL);
2758         CHECK(n_tokens == 3, EINVAL);
2759
2760         h = header_parse(p, dst);
2761         CHECK(h, EINVAL);
2762
2763         tf = action_field_parse(action, src);
2764         CHECK(tf, EINVAL);
2765
2766         instr->type = INSTR_DMA_HT;
2767         instr->dma.dst.header_id[0] = h->id;
2768         instr->dma.dst.struct_id[0] = h->struct_id;
2769         instr->dma.n_bytes[0] = h->st->n_bits / 8;
2770         instr->dma.src.offset[0] = tf->offset / 8;
2771
2772         return 0;
2773 }
2774
2775 static inline void
2776 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2777
2778 static inline void
2779 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2780 {
2781         struct thread *t = &p->threads[p->thread_id];
2782         struct instruction *ip = t->ip;
2783         uint8_t *action_data = t->structs[0];
2784         uint64_t valid_headers = t->valid_headers;
2785         uint32_t i;
2786
2787         for (i = 0; i < n_dma; i++) {
2788                 uint32_t header_id = ip->dma.dst.header_id[i];
2789                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2790                 uint32_t offset = ip->dma.src.offset[i];
2791                 uint32_t n_bytes = ip->dma.n_bytes[i];
2792
2793                 struct header_runtime *h = &t->headers[header_id];
2794                 uint8_t *h_ptr0 = h->ptr0;
2795                 uint8_t *h_ptr = t->structs[struct_id];
2796
2797                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2798                         h_ptr : h_ptr0;
2799                 void *src = &action_data[offset];
2800
2801                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2802
2803                 /* Headers. */
2804                 memcpy(dst, src, n_bytes);
2805                 t->structs[struct_id] = dst;
2806                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2807         }
2808
2809         t->valid_headers = valid_headers;
2810 }
2811
2812 static inline void
2813 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2814 {
2815         __instr_dma_ht_exec(p, 1);
2816
2817         /* Thread. */
2818         thread_ip_inc(p);
2819 }
2820
2821 static inline void
2822 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2823 {
2824         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2825               p->thread_id);
2826
2827         __instr_dma_ht_exec(p, 2);
2828
2829         /* Thread. */
2830         thread_ip_inc(p);
2831 }
2832
2833 static inline void
2834 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2835 {
2836         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2837               p->thread_id);
2838
2839         __instr_dma_ht_exec(p, 3);
2840
2841         /* Thread. */
2842         thread_ip_inc(p);
2843 }
2844
2845 static inline void
2846 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2847 {
2848         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2849               p->thread_id);
2850
2851         __instr_dma_ht_exec(p, 4);
2852
2853         /* Thread. */
2854         thread_ip_inc(p);
2855 }
2856
2857 static inline void
2858 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2859 {
2860         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2861               p->thread_id);
2862
2863         __instr_dma_ht_exec(p, 5);
2864
2865         /* Thread. */
2866         thread_ip_inc(p);
2867 }
2868
2869 static inline void
2870 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2871 {
2872         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2873               p->thread_id);
2874
2875         __instr_dma_ht_exec(p, 6);
2876
2877         /* Thread. */
2878         thread_ip_inc(p);
2879 }
2880
2881 static inline void
2882 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2883 {
2884         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2885               p->thread_id);
2886
2887         __instr_dma_ht_exec(p, 7);
2888
2889         /* Thread. */
2890         thread_ip_inc(p);
2891 }
2892
2893 static inline void
2894 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2895 {
2896         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2897               p->thread_id);
2898
2899         __instr_dma_ht_exec(p, 8);
2900
2901         /* Thread. */
2902         thread_ip_inc(p);
2903 }
2904
2905 /*
2906  * alu.
2907  */
2908 static int
2909 instr_alu_add_translate(struct rte_swx_pipeline *p,
2910                         struct action *action,
2911                         char **tokens,
2912                         int n_tokens,
2913                         struct instruction *instr,
2914                         struct instruction_data *data __rte_unused)
2915 {
2916         char *dst = tokens[1], *src = tokens[2];
2917         struct field *fdst, *fsrc;
2918         uint32_t dst_struct_id, src_struct_id, src_val;
2919
2920         CHECK(n_tokens == 3, EINVAL);
2921
2922         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2923         CHECK(fdst, EINVAL);
2924
2925         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2926         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2927         if (fsrc) {
2928                 instr->type = INSTR_ALU_ADD;
2929                 if (dst[0] == 'h' && src[0] == 'm')
2930                         instr->type = INSTR_ALU_ADD_HM;
2931                 if (dst[0] == 'm' && src[0] == 'h')
2932                         instr->type = INSTR_ALU_ADD_MH;
2933                 if (dst[0] == 'h' && src[0] == 'h')
2934                         instr->type = INSTR_ALU_ADD_HH;
2935
2936                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2937                 instr->alu.dst.n_bits = fdst->n_bits;
2938                 instr->alu.dst.offset = fdst->offset / 8;
2939                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2940                 instr->alu.src.n_bits = fsrc->n_bits;
2941                 instr->alu.src.offset = fsrc->offset / 8;
2942                 return 0;
2943         }
2944
2945         /* ADD_MI, ADD_HI. */
2946         src_val = strtoul(src, &src, 0);
2947         CHECK(!src[0], EINVAL);
2948
2949         instr->type = INSTR_ALU_ADD_MI;
2950         if (dst[0] == 'h')
2951                 instr->type = INSTR_ALU_ADD_HI;
2952
2953         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2954         instr->alu.dst.n_bits = fdst->n_bits;
2955         instr->alu.dst.offset = fdst->offset / 8;
2956         instr->alu.src_val = (uint32_t)src_val;
2957         return 0;
2958 }
2959
2960 static int
2961 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2962                         struct action *action,
2963                         char **tokens,
2964                         int n_tokens,
2965                         struct instruction *instr,
2966                         struct instruction_data *data __rte_unused)
2967 {
2968         char *dst = tokens[1], *src = tokens[2];
2969         struct field *fdst, *fsrc;
2970         uint32_t dst_struct_id, src_struct_id, src_val;
2971
2972         CHECK(n_tokens == 3, EINVAL);
2973
2974         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2975         CHECK(fdst, EINVAL);
2976
2977         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2978         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2979         if (fsrc) {
2980                 instr->type = INSTR_ALU_SUB;
2981                 if (dst[0] == 'h' && src[0] == 'm')
2982                         instr->type = INSTR_ALU_SUB_HM;
2983                 if (dst[0] == 'm' && src[0] == 'h')
2984                         instr->type = INSTR_ALU_SUB_MH;
2985                 if (dst[0] == 'h' && src[0] == 'h')
2986                         instr->type = INSTR_ALU_SUB_HH;
2987
2988                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2989                 instr->alu.dst.n_bits = fdst->n_bits;
2990                 instr->alu.dst.offset = fdst->offset / 8;
2991                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2992                 instr->alu.src.n_bits = fsrc->n_bits;
2993                 instr->alu.src.offset = fsrc->offset / 8;
2994                 return 0;
2995         }
2996
2997         /* SUB_MI, SUB_HI. */
2998         src_val = strtoul(src, &src, 0);
2999         CHECK(!src[0], EINVAL);
3000
3001         instr->type = INSTR_ALU_SUB_MI;
3002         if (dst[0] == 'h')
3003                 instr->type = INSTR_ALU_SUB_HI;
3004
3005         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3006         instr->alu.dst.n_bits = fdst->n_bits;
3007         instr->alu.dst.offset = fdst->offset / 8;
3008         instr->alu.src_val = (uint32_t)src_val;
3009         return 0;
3010 }
3011
3012 static int
3013 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3014                           struct action *action __rte_unused,
3015                           char **tokens,
3016                           int n_tokens,
3017                           struct instruction *instr,
3018                           struct instruction_data *data __rte_unused)
3019 {
3020         char *dst = tokens[1], *src = tokens[2];
3021         struct header *hdst, *hsrc;
3022         struct field *fdst, *fsrc;
3023
3024         CHECK(n_tokens == 3, EINVAL);
3025
3026         fdst = header_field_parse(p, dst, &hdst);
3027         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3028
3029         /* CKADD_FIELD. */
3030         fsrc = header_field_parse(p, src, &hsrc);
3031         if (fsrc) {
3032                 instr->type = INSTR_ALU_CKADD_FIELD;
3033                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3034                 instr->alu.dst.n_bits = fdst->n_bits;
3035                 instr->alu.dst.offset = fdst->offset / 8;
3036                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3037                 instr->alu.src.n_bits = fsrc->n_bits;
3038                 instr->alu.src.offset = fsrc->offset / 8;
3039                 return 0;
3040         }
3041
3042         /* CKADD_STRUCT, CKADD_STRUCT20. */
3043         hsrc = header_parse(p, src);
3044         CHECK(hsrc, EINVAL);
3045
3046         instr->type = INSTR_ALU_CKADD_STRUCT;
3047         if ((hsrc->st->n_bits / 8) == 20)
3048                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3049
3050         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3051         instr->alu.dst.n_bits = fdst->n_bits;
3052         instr->alu.dst.offset = fdst->offset / 8;
3053         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3054         instr->alu.src.n_bits = hsrc->st->n_bits;
3055         instr->alu.src.offset = 0; /* Unused. */
3056         return 0;
3057 }
3058
3059 static int
3060 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3061                           struct action *action __rte_unused,
3062                           char **tokens,
3063                           int n_tokens,
3064                           struct instruction *instr,
3065                           struct instruction_data *data __rte_unused)
3066 {
3067         char *dst = tokens[1], *src = tokens[2];
3068         struct header *hdst, *hsrc;
3069         struct field *fdst, *fsrc;
3070
3071         CHECK(n_tokens == 3, EINVAL);
3072
3073         fdst = header_field_parse(p, dst, &hdst);
3074         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3075
3076         fsrc = header_field_parse(p, src, &hsrc);
3077         CHECK(fsrc, EINVAL);
3078
3079         instr->type = INSTR_ALU_CKSUB_FIELD;
3080         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3081         instr->alu.dst.n_bits = fdst->n_bits;
3082         instr->alu.dst.offset = fdst->offset / 8;
3083         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3084         instr->alu.src.n_bits = fsrc->n_bits;
3085         instr->alu.src.offset = fsrc->offset / 8;
3086         return 0;
3087 }
3088
3089 static int
3090 instr_alu_and_translate(struct rte_swx_pipeline *p,
3091                         struct action *action,
3092                         char **tokens,
3093                         int n_tokens,
3094                         struct instruction *instr,
3095                         struct instruction_data *data __rte_unused)
3096 {
3097         char *dst = tokens[1], *src = tokens[2];
3098         struct field *fdst, *fsrc;
3099         uint32_t dst_struct_id, src_struct_id, src_val;
3100
3101         CHECK(n_tokens == 3, EINVAL);
3102
3103         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3104         CHECK(fdst, EINVAL);
3105
3106         /* AND or AND_S. */
3107         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3108         if (fsrc) {
3109                 instr->type = INSTR_ALU_AND;
3110                 if ((dst[0] == 'h' && src[0] != 'h') ||
3111                     (dst[0] != 'h' && src[0] == 'h'))
3112                         instr->type = INSTR_ALU_AND_S;
3113
3114                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3115                 instr->alu.dst.n_bits = fdst->n_bits;
3116                 instr->alu.dst.offset = fdst->offset / 8;
3117                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3118                 instr->alu.src.n_bits = fsrc->n_bits;
3119                 instr->alu.src.offset = fsrc->offset / 8;
3120                 return 0;
3121         }
3122
3123         /* AND_I. */
3124         src_val = strtoul(src, &src, 0);
3125         CHECK(!src[0], EINVAL);
3126
3127         if (dst[0] == 'h')
3128                 src_val = htonl(src_val);
3129
3130         instr->type = INSTR_ALU_AND_I;
3131         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3132         instr->alu.dst.n_bits = fdst->n_bits;
3133         instr->alu.dst.offset = fdst->offset / 8;
3134         instr->alu.src_val = (uint32_t)src_val;
3135         return 0;
3136 }
3137
3138 static int
3139 instr_alu_or_translate(struct rte_swx_pipeline *p,
3140                        struct action *action,
3141                        char **tokens,
3142                        int n_tokens,
3143                        struct instruction *instr,
3144                        struct instruction_data *data __rte_unused)
3145 {
3146         char *dst = tokens[1], *src = tokens[2];
3147         struct field *fdst, *fsrc;
3148         uint32_t dst_struct_id, src_struct_id, src_val;
3149
3150         CHECK(n_tokens == 3, EINVAL);
3151
3152         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3153         CHECK(fdst, EINVAL);
3154
3155         /* OR or OR_S. */
3156         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3157         if (fsrc) {
3158                 instr->type = INSTR_ALU_OR;
3159                 if ((dst[0] == 'h' && src[0] != 'h') ||
3160                     (dst[0] != 'h' && src[0] == 'h'))
3161                         instr->type = INSTR_ALU_OR_S;
3162
3163                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3164                 instr->alu.dst.n_bits = fdst->n_bits;
3165                 instr->alu.dst.offset = fdst->offset / 8;
3166                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3167                 instr->alu.src.n_bits = fsrc->n_bits;
3168                 instr->alu.src.offset = fsrc->offset / 8;
3169                 return 0;
3170         }
3171
3172         /* OR_I. */
3173         src_val = strtoul(src, &src, 0);
3174         CHECK(!src[0], EINVAL);
3175
3176         if (dst[0] == 'h')
3177                 src_val = htonl(src_val);
3178
3179         instr->type = INSTR_ALU_OR_I;
3180         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3181         instr->alu.dst.n_bits = fdst->n_bits;
3182         instr->alu.dst.offset = fdst->offset / 8;
3183         instr->alu.src_val = (uint32_t)src_val;
3184         return 0;
3185 }
3186
3187 static inline void
3188 instr_alu_add_exec(struct rte_swx_pipeline *p)
3189 {
3190         struct thread *t = &p->threads[p->thread_id];
3191         struct instruction *ip = t->ip;
3192
3193         TRACE("[Thread %2u] add\n", p->thread_id);
3194
3195         /* Structs. */
3196         ALU(t, ip, +);
3197
3198         /* Thread. */
3199         thread_ip_inc(p);
3200 }
3201
3202 static inline void
3203 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3204 {
3205         struct thread *t = &p->threads[p->thread_id];
3206         struct instruction *ip = t->ip;
3207
3208         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3209
3210         /* Structs. */
3211         ALU_MH(t, ip, +);
3212
3213         /* Thread. */
3214         thread_ip_inc(p);
3215 }
3216
3217 static inline void
3218 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3219 {
3220         struct thread *t = &p->threads[p->thread_id];
3221         struct instruction *ip = t->ip;
3222
3223         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3224
3225         /* Structs. */
3226         ALU_HM(t, ip, +);
3227
3228         /* Thread. */
3229         thread_ip_inc(p);
3230 }
3231
3232 static inline void
3233 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3234 {
3235         struct thread *t = &p->threads[p->thread_id];
3236         struct instruction *ip = t->ip;
3237
3238         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3239
3240         /* Structs. */
3241         ALU_HH(t, ip, +);
3242
3243         /* Thread. */
3244         thread_ip_inc(p);
3245 }
3246
3247 static inline void
3248 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3249 {
3250         struct thread *t = &p->threads[p->thread_id];
3251         struct instruction *ip = t->ip;
3252
3253         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3254
3255         /* Structs. */
3256         ALU_MI(t, ip, +);
3257
3258         /* Thread. */
3259         thread_ip_inc(p);
3260 }
3261
3262 static inline void
3263 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3264 {
3265         struct thread *t = &p->threads[p->thread_id];
3266         struct instruction *ip = t->ip;
3267
3268         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3269
3270         /* Structs. */
3271         ALU_HI(t, ip, +);
3272
3273         /* Thread. */
3274         thread_ip_inc(p);
3275 }
3276
3277 static inline void
3278 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3279 {
3280         struct thread *t = &p->threads[p->thread_id];
3281         struct instruction *ip = t->ip;
3282
3283         TRACE("[Thread %2u] sub\n", p->thread_id);
3284
3285         /* Structs. */
3286         ALU(t, ip, -);
3287
3288         /* Thread. */
3289         thread_ip_inc(p);
3290 }
3291
3292 static inline void
3293 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3294 {
3295         struct thread *t = &p->threads[p->thread_id];
3296         struct instruction *ip = t->ip;
3297
3298         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3299
3300         /* Structs. */
3301         ALU_MH(t, ip, -);
3302
3303         /* Thread. */
3304         thread_ip_inc(p);
3305 }
3306
3307 static inline void
3308 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3309 {
3310         struct thread *t = &p->threads[p->thread_id];
3311         struct instruction *ip = t->ip;
3312
3313         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3314
3315         /* Structs. */
3316         ALU_HM(t, ip, -);
3317
3318         /* Thread. */
3319         thread_ip_inc(p);
3320 }
3321
3322 static inline void
3323 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3324 {
3325         struct thread *t = &p->threads[p->thread_id];
3326         struct instruction *ip = t->ip;
3327
3328         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3329
3330         /* Structs. */
3331         ALU_HH(t, ip, -);
3332
3333         /* Thread. */
3334         thread_ip_inc(p);
3335 }
3336
3337 static inline void
3338 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3339 {
3340         struct thread *t = &p->threads[p->thread_id];
3341         struct instruction *ip = t->ip;
3342
3343         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3344
3345         /* Structs. */
3346         ALU_MI(t, ip, -);
3347
3348         /* Thread. */
3349         thread_ip_inc(p);
3350 }
3351
3352 static inline void
3353 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3354 {
3355         struct thread *t = &p->threads[p->thread_id];
3356         struct instruction *ip = t->ip;
3357
3358         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3359
3360         /* Structs. */
3361         ALU_HI(t, ip, -);
3362
3363         /* Thread. */
3364         thread_ip_inc(p);
3365 }
3366
3367 static inline void
3368 instr_alu_and_exec(struct rte_swx_pipeline *p)
3369 {
3370         struct thread *t = &p->threads[p->thread_id];
3371         struct instruction *ip = t->ip;
3372
3373         TRACE("[Thread %2u] and\n", p->thread_id);
3374
3375         /* Structs. */
3376         ALU(t, ip, &);
3377
3378         /* Thread. */
3379         thread_ip_inc(p);
3380 }
3381
3382 static inline void
3383 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
3384 {
3385         struct thread *t = &p->threads[p->thread_id];
3386         struct instruction *ip = t->ip;
3387
3388         TRACE("[Thread %2u] and (s)\n", p->thread_id);
3389
3390         /* Structs. */
3391         ALU_S(t, ip, &);
3392
3393         /* Thread. */
3394         thread_ip_inc(p);
3395 }
3396
3397 static inline void
3398 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3399 {
3400         struct thread *t = &p->threads[p->thread_id];
3401         struct instruction *ip = t->ip;
3402
3403         TRACE("[Thread %2u] and (i)\n", p->thread_id);
3404
3405         /* Structs. */
3406         ALU_I(t, ip, &);
3407
3408         /* Thread. */
3409         thread_ip_inc(p);
3410 }
3411
3412 static inline void
3413 instr_alu_or_exec(struct rte_swx_pipeline *p)
3414 {
3415         struct thread *t = &p->threads[p->thread_id];
3416         struct instruction *ip = t->ip;
3417
3418         TRACE("[Thread %2u] or\n", p->thread_id);
3419
3420         /* Structs. */
3421         ALU(t, ip, |);
3422
3423         /* Thread. */
3424         thread_ip_inc(p);
3425 }
3426
3427 static inline void
3428 instr_alu_or_s_exec(struct rte_swx_pipeline *p)
3429 {
3430         struct thread *t = &p->threads[p->thread_id];
3431         struct instruction *ip = t->ip;
3432
3433         TRACE("[Thread %2u] or (s)\n", p->thread_id);
3434
3435         /* Structs. */
3436         ALU_S(t, ip, |);
3437
3438         /* Thread. */
3439         thread_ip_inc(p);
3440 }
3441
3442 static inline void
3443 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3444 {
3445         struct thread *t = &p->threads[p->thread_id];
3446         struct instruction *ip = t->ip;
3447
3448         TRACE("[Thread %2u] or (i)\n", p->thread_id);
3449
3450         /* Structs. */
3451         ALU_I(t, ip, |);
3452
3453         /* Thread. */
3454         thread_ip_inc(p);
3455 }
3456
3457 static inline void
3458 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3459 {
3460         struct thread *t = &p->threads[p->thread_id];
3461         struct instruction *ip = t->ip;
3462         uint8_t *dst_struct, *src_struct;
3463         uint16_t *dst16_ptr, dst;
3464         uint64_t *src64_ptr, src64, src64_mask, src;
3465         uint64_t r;
3466
3467         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3468
3469         /* Structs. */
3470         dst_struct = t->structs[ip->alu.dst.struct_id];
3471         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3472         dst = *dst16_ptr;
3473
3474         src_struct = t->structs[ip->alu.src.struct_id];
3475         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3476         src64 = *src64_ptr;
3477         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3478         src = src64 & src64_mask;
3479
3480         r = dst;
3481         r = ~r & 0xFFFF;
3482
3483         /* The first input (r) is a 16-bit number. The second and the third
3484          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3485          * three numbers (output r) is a 34-bit number.
3486          */
3487         r += (src >> 32) + (src & 0xFFFFFFFF);
3488
3489         /* The first input is a 16-bit number. The second input is an 18-bit
3490          * number. In the worst case scenario, the sum of the two numbers is a
3491          * 19-bit number.
3492          */
3493         r = (r & 0xFFFF) + (r >> 16);
3494
3495         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3496          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3497          */
3498         r = (r & 0xFFFF) + (r >> 16);
3499
3500         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3501          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3502          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3503          * therefore the output r is always a 16-bit number.
3504          */
3505         r = (r & 0xFFFF) + (r >> 16);
3506
3507         r = ~r & 0xFFFF;
3508         r = r ? r : 0xFFFF;
3509
3510         *dst16_ptr = (uint16_t)r;
3511
3512         /* Thread. */
3513         thread_ip_inc(p);
3514 }
3515
3516 static inline void
3517 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3518 {
3519         struct thread *t = &p->threads[p->thread_id];
3520         struct instruction *ip = t->ip;
3521         uint8_t *dst_struct, *src_struct;
3522         uint16_t *dst16_ptr, dst;
3523         uint64_t *src64_ptr, src64, src64_mask, src;
3524         uint64_t r;
3525
3526         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3527
3528         /* Structs. */
3529         dst_struct = t->structs[ip->alu.dst.struct_id];
3530         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3531         dst = *dst16_ptr;
3532
3533         src_struct = t->structs[ip->alu.src.struct_id];
3534         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3535         src64 = *src64_ptr;
3536         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3537         src = src64 & src64_mask;
3538
3539         r = dst;
3540         r = ~r & 0xFFFF;
3541
3542         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3543          * the following sequence of operations in 2's complement arithmetic:
3544          *    a '- b = (a - b) % 0xFFFF.
3545          *
3546          * In order to prevent an underflow for the below subtraction, in which
3547          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3548          * minuend), we first add a multiple of the 0xFFFF modulus to the
3549          * minuend. The number we add to the minuend needs to be a 34-bit number
3550          * or higher, so for readability reasons we picked the 36-bit multiple.
3551          * We are effectively turning the 16-bit minuend into a 36-bit number:
3552          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3553          */
3554         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3555
3556         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3557          * result (the output r) is a 36-bit number.
3558          */
3559         r -= (src >> 32) + (src & 0xFFFFFFFF);
3560
3561         /* The first input is a 16-bit number. The second input is a 20-bit
3562          * number. Their sum is a 21-bit number.
3563          */
3564         r = (r & 0xFFFF) + (r >> 16);
3565
3566         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3567          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3568          */
3569         r = (r & 0xFFFF) + (r >> 16);
3570
3571         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3572          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3573          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3574          * generated, therefore the output r is always a 16-bit number.
3575          */
3576         r = (r & 0xFFFF) + (r >> 16);
3577
3578         r = ~r & 0xFFFF;
3579         r = r ? r : 0xFFFF;
3580
3581         *dst16_ptr = (uint16_t)r;
3582
3583         /* Thread. */
3584         thread_ip_inc(p);
3585 }
3586
3587 static inline void
3588 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3589 {
3590         struct thread *t = &p->threads[p->thread_id];
3591         struct instruction *ip = t->ip;
3592         uint8_t *dst_struct, *src_struct;
3593         uint16_t *dst16_ptr;
3594         uint32_t *src32_ptr;
3595         uint64_t r0, r1;
3596
3597         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3598
3599         /* Structs. */
3600         dst_struct = t->structs[ip->alu.dst.struct_id];
3601         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3602
3603         src_struct = t->structs[ip->alu.src.struct_id];
3604         src32_ptr = (uint32_t *)&src_struct[0];
3605
3606         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
3607         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3608         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
3609         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3610         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3611
3612         /* The first input is a 16-bit number. The second input is a 19-bit
3613          * number. Their sum is a 20-bit number.
3614          */
3615         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3616
3617         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3618          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3619          */
3620         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3621
3622         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3623          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3624          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3625          * generated, therefore the output r is always a 16-bit number.
3626          */
3627         r0 = (r0 & 0xFFFF) + (r0 >> 16);
3628
3629         r0 = ~r0 & 0xFFFF;
3630         r0 = r0 ? r0 : 0xFFFF;
3631
3632         *dst16_ptr = (uint16_t)r0;
3633
3634         /* Thread. */
3635         thread_ip_inc(p);
3636 }
3637
3638 static inline void
3639 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3640 {
3641         struct thread *t = &p->threads[p->thread_id];
3642         struct instruction *ip = t->ip;
3643         uint8_t *dst_struct, *src_struct;
3644         uint16_t *dst16_ptr;
3645         uint32_t *src32_ptr;
3646         uint64_t r = 0;
3647         uint32_t i;
3648
3649         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3650
3651         /* Structs. */
3652         dst_struct = t->structs[ip->alu.dst.struct_id];
3653         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3654
3655         src_struct = t->structs[ip->alu.src.struct_id];
3656         src32_ptr = (uint32_t *)&src_struct[0];
3657
3658         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
3659          * Therefore, in the worst case scenario, a 35-bit number is added to a
3660          * 16-bit number (the input r), so the output r is 36-bit number.
3661          */
3662         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
3663                 r += *src32_ptr;
3664
3665         /* The first input is a 16-bit number. The second input is a 20-bit
3666          * number. Their sum is a 21-bit number.
3667          */
3668         r = (r & 0xFFFF) + (r >> 16);
3669
3670         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3671          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
3672          */
3673         r = (r & 0xFFFF) + (r >> 16);
3674
3675         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3676          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3677          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3678          * generated, therefore the output r is always a 16-bit number.
3679          */
3680         r = (r & 0xFFFF) + (r >> 16);
3681
3682         r = ~r & 0xFFFF;
3683         r = r ? r : 0xFFFF;
3684
3685         *dst16_ptr = (uint16_t)r;
3686
3687         /* Thread. */
3688         thread_ip_inc(p);
3689 }
3690
3691 #define RTE_SWX_INSTRUCTION_TOKENS_MAX 16
3692
3693 static int
3694 instr_translate(struct rte_swx_pipeline *p,
3695                 struct action *action,
3696                 char *string,
3697                 struct instruction *instr,
3698                 struct instruction_data *data)
3699 {
3700         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
3701         int n_tokens = 0, tpos = 0;
3702
3703         /* Parse the instruction string into tokens. */
3704         for ( ; ; ) {
3705                 char *token;
3706
3707                 token = strtok_r(string, " \t\v", &string);
3708                 if (!token)
3709                         break;
3710
3711                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
3712
3713                 tokens[n_tokens] = token;
3714                 n_tokens++;
3715         }
3716
3717         CHECK(n_tokens, EINVAL);
3718
3719         /* Handle the optional instruction label. */
3720         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
3721                 strcpy(data->label, tokens[0]);
3722
3723                 tpos += 2;
3724                 CHECK(n_tokens - tpos, EINVAL);
3725         }
3726
3727         /* Identify the instruction type. */
3728         if (!strcmp(tokens[tpos], "rx"))
3729                 return instr_rx_translate(p,
3730                                           action,
3731                                           &tokens[tpos],
3732                                           n_tokens - tpos,
3733                                           instr,
3734                                           data);
3735
3736         if (!strcmp(tokens[tpos], "tx"))
3737                 return instr_tx_translate(p,
3738                                           action,
3739                                           &tokens[tpos],
3740                                           n_tokens - tpos,
3741                                           instr,
3742                                           data);
3743
3744         if (!strcmp(tokens[tpos], "extract"))
3745                 return instr_hdr_extract_translate(p,
3746                                                    action,
3747                                                    &tokens[tpos],
3748                                                    n_tokens - tpos,
3749                                                    instr,
3750                                                    data);
3751
3752         if (!strcmp(tokens[tpos], "emit"))
3753                 return instr_hdr_emit_translate(p,
3754                                                 action,
3755                                                 &tokens[tpos],
3756                                                 n_tokens - tpos,
3757                                                 instr,
3758                                                 data);
3759
3760         if (!strcmp(tokens[tpos], "validate"))
3761                 return instr_hdr_validate_translate(p,
3762                                                     action,
3763                                                     &tokens[tpos],
3764                                                     n_tokens - tpos,
3765                                                     instr,
3766                                                     data);
3767
3768         if (!strcmp(tokens[tpos], "invalidate"))
3769                 return instr_hdr_invalidate_translate(p,
3770                                                       action,
3771                                                       &tokens[tpos],
3772                                                       n_tokens - tpos,
3773                                                       instr,
3774                                                       data);
3775
3776         if (!strcmp(tokens[tpos], "mov"))
3777                 return instr_mov_translate(p,
3778                                            action,
3779                                            &tokens[tpos],
3780                                            n_tokens - tpos,
3781                                            instr,
3782                                            data);
3783
3784         if (!strcmp(tokens[tpos], "dma"))
3785                 return instr_dma_translate(p,
3786                                            action,
3787                                            &tokens[tpos],
3788                                            n_tokens - tpos,
3789                                            instr,
3790                                            data);
3791
3792         if (!strcmp(tokens[tpos], "add"))
3793                 return instr_alu_add_translate(p,
3794                                                action,
3795                                                &tokens[tpos],
3796                                                n_tokens - tpos,
3797                                                instr,
3798                                                data);
3799
3800         if (!strcmp(tokens[tpos], "sub"))
3801                 return instr_alu_sub_translate(p,
3802                                                action,
3803                                                &tokens[tpos],
3804                                                n_tokens - tpos,
3805                                                instr,
3806                                                data);
3807
3808         if (!strcmp(tokens[tpos], "ckadd"))
3809                 return instr_alu_ckadd_translate(p,
3810                                                  action,
3811                                                  &tokens[tpos],
3812                                                  n_tokens - tpos,
3813                                                  instr,
3814                                                  data);
3815
3816         if (!strcmp(tokens[tpos], "cksub"))
3817                 return instr_alu_cksub_translate(p,
3818                                                  action,
3819                                                  &tokens[tpos],
3820                                                  n_tokens - tpos,
3821                                                  instr,
3822                                                  data);
3823
3824         if (!strcmp(tokens[tpos], "and"))
3825                 return instr_alu_and_translate(p,
3826                                                action,
3827                                                &tokens[tpos],
3828                                                n_tokens - tpos,
3829                                                instr,
3830                                                data);
3831
3832         if (!strcmp(tokens[tpos], "or"))
3833                 return instr_alu_or_translate(p,
3834                                               action,
3835                                               &tokens[tpos],
3836                                               n_tokens - tpos,
3837                                               instr,
3838                                               data);
3839
3840         CHECK(0, EINVAL);
3841 }
3842
3843 static uint32_t
3844 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
3845 {
3846         uint32_t count = 0, i;
3847
3848         if (!label[0])
3849                 return 0;
3850
3851         for (i = 0; i < n; i++)
3852                 if (!strcmp(label, data[i].jmp_label))
3853                         count++;
3854
3855         return count;
3856 }
3857
3858 static int
3859 instr_label_check(struct instruction_data *instruction_data,
3860                   uint32_t n_instructions)
3861 {
3862         uint32_t i;
3863
3864         /* Check that all instruction labels are unique. */
3865         for (i = 0; i < n_instructions; i++) {
3866                 struct instruction_data *data = &instruction_data[i];
3867                 char *label = data->label;
3868                 uint32_t j;
3869
3870                 if (!label[0])
3871                         continue;
3872
3873                 for (j = i + 1; j < n_instructions; j++)
3874                         CHECK(strcmp(label, data[j].label), EINVAL);
3875         }
3876
3877         /* Get users for each instruction label. */
3878         for (i = 0; i < n_instructions; i++) {
3879                 struct instruction_data *data = &instruction_data[i];
3880                 char *label = data->label;
3881
3882                 data->n_users = label_is_used(instruction_data,
3883                                               n_instructions,
3884                                               label);
3885         }
3886
3887         return 0;
3888 }
3889
3890 static int
3891 instruction_config(struct rte_swx_pipeline *p,
3892                    struct action *a,
3893                    const char **instructions,
3894                    uint32_t n_instructions)
3895 {
3896         struct instruction *instr = NULL;
3897         struct instruction_data *data = NULL;
3898         char *string = NULL;
3899         int err = 0;
3900         uint32_t i;
3901
3902         CHECK(n_instructions, EINVAL);
3903         CHECK(instructions, EINVAL);
3904         for (i = 0; i < n_instructions; i++)
3905                 CHECK(instructions[i], EINVAL);
3906
3907         /* Memory allocation. */
3908         instr = calloc(n_instructions, sizeof(struct instruction));
3909         if (!instr) {
3910                 err = ENOMEM;
3911                 goto error;
3912         }
3913
3914         data = calloc(n_instructions, sizeof(struct instruction_data));
3915         if (!data) {
3916                 err = ENOMEM;
3917                 goto error;
3918         }
3919
3920         for (i = 0; i < n_instructions; i++) {
3921                 string = strdup(instructions[i]);
3922                 if (!string) {
3923                         err = ENOMEM;
3924                         goto error;
3925                 }
3926
3927                 err = instr_translate(p, a, string, &instr[i], &data[i]);
3928                 if (err)
3929                         goto error;
3930
3931                 free(string);
3932         }
3933
3934         err = instr_label_check(data, n_instructions);
3935         if (err)
3936                 goto error;
3937
3938         free(data);
3939
3940         if (a) {
3941                 a->instructions = instr;
3942                 a->n_instructions = n_instructions;
3943         } else {
3944                 p->instructions = instr;
3945                 p->n_instructions = n_instructions;
3946         }
3947
3948         return 0;
3949
3950 error:
3951         free(string);
3952         free(data);
3953         free(instr);
3954         return err;
3955 }
3956
3957 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
3958
3959 static instr_exec_t instruction_table[] = {
3960         [INSTR_RX] = instr_rx_exec,
3961         [INSTR_TX] = instr_tx_exec,
3962
3963         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
3964         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
3965         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
3966         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
3967         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
3968         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
3969         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
3970         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
3971
3972         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
3973         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
3974         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
3975         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
3976         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
3977         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
3978         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
3979         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
3980         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
3981
3982         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
3983         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
3984
3985         [INSTR_MOV] = instr_mov_exec,
3986         [INSTR_MOV_S] = instr_mov_s_exec,
3987         [INSTR_MOV_I] = instr_mov_i_exec,
3988
3989         [INSTR_DMA_HT] = instr_dma_ht_exec,
3990         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
3991         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
3992         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
3993         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
3994         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
3995         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
3996         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
3997
3998         [INSTR_ALU_ADD] = instr_alu_add_exec,
3999         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
4000         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
4001         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
4002         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
4003         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
4004
4005         [INSTR_ALU_SUB] = instr_alu_sub_exec,
4006         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
4007         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
4008         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
4009         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
4010         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
4011
4012         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
4013         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
4014         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
4015         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
4016
4017         [INSTR_ALU_AND] = instr_alu_and_exec,
4018         [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
4019         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
4020
4021         [INSTR_ALU_OR] = instr_alu_or_exec,
4022         [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
4023         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
4024 };
4025
4026 static inline void
4027 instr_exec(struct rte_swx_pipeline *p)
4028 {
4029         struct thread *t = &p->threads[p->thread_id];
4030         struct instruction *ip = t->ip;
4031         instr_exec_t instr = instruction_table[ip->type];
4032
4033         instr(p);
4034 }
4035
4036 /*
4037  * Action.
4038  */
4039 static struct action *
4040 action_find(struct rte_swx_pipeline *p, const char *name)
4041 {
4042         struct action *elem;
4043
4044         if (!name)
4045                 return NULL;
4046
4047         TAILQ_FOREACH(elem, &p->actions, node)
4048                 if (strcmp(elem->name, name) == 0)
4049                         return elem;
4050
4051         return NULL;
4052 }
4053
4054 static struct field *
4055 action_field_find(struct action *a, const char *name)
4056 {
4057         return a->st ? struct_type_field_find(a->st, name) : NULL;
4058 }
4059
4060 static struct field *
4061 action_field_parse(struct action *action, const char *name)
4062 {
4063         if (name[0] != 't' || name[1] != '.')
4064                 return NULL;
4065
4066         return action_field_find(action, &name[2]);
4067 }
4068
4069 int
4070 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
4071                                const char *name,
4072                                const char *args_struct_type_name,
4073                                const char **instructions,
4074                                uint32_t n_instructions)
4075 {
4076         struct struct_type *args_struct_type;
4077         struct action *a;
4078         int err;
4079
4080         CHECK(p, EINVAL);
4081
4082         CHECK_NAME(name, EINVAL);
4083         CHECK(!action_find(p, name), EEXIST);
4084
4085         if (args_struct_type_name) {
4086                 CHECK_NAME(args_struct_type_name, EINVAL);
4087                 args_struct_type = struct_type_find(p, args_struct_type_name);
4088                 CHECK(args_struct_type, EINVAL);
4089         } else {
4090                 args_struct_type = NULL;
4091         }
4092
4093         /* Node allocation. */
4094         a = calloc(1, sizeof(struct action));
4095         CHECK(a, ENOMEM);
4096
4097         /* Node initialization. */
4098         strcpy(a->name, name);
4099         a->st = args_struct_type;
4100         a->id = p->n_actions;
4101
4102         /* Instruction translation. */
4103         err = instruction_config(p, a, instructions, n_instructions);
4104         if (err) {
4105                 free(a);
4106                 return err;
4107         }
4108
4109         /* Node add to tailq. */
4110         TAILQ_INSERT_TAIL(&p->actions, a, node);
4111         p->n_actions++;
4112
4113         return 0;
4114 }
4115
4116 static int
4117 action_build(struct rte_swx_pipeline *p)
4118 {
4119         struct action *action;
4120
4121         p->action_instructions = calloc(p->n_actions,
4122                                         sizeof(struct instruction *));
4123         CHECK(p->action_instructions, ENOMEM);
4124
4125         TAILQ_FOREACH(action, &p->actions, node)
4126                 p->action_instructions[action->id] = action->instructions;
4127
4128         return 0;
4129 }
4130
4131 static void
4132 action_build_free(struct rte_swx_pipeline *p)
4133 {
4134         free(p->action_instructions);
4135         p->action_instructions = NULL;
4136 }
4137
4138 static void
4139 action_free(struct rte_swx_pipeline *p)
4140 {
4141         action_build_free(p);
4142
4143         for ( ; ; ) {
4144                 struct action *action;
4145
4146                 action = TAILQ_FIRST(&p->actions);
4147                 if (!action)
4148                         break;
4149
4150                 TAILQ_REMOVE(&p->actions, action, node);
4151                 free(action->instructions);
4152                 free(action);
4153         }
4154 }
4155
4156 /*
4157  * Table.
4158  */
4159 static struct table_type *
4160 table_type_find(struct rte_swx_pipeline *p, const char *name)
4161 {
4162         struct table_type *elem;
4163
4164         TAILQ_FOREACH(elem, &p->table_types, node)
4165                 if (strcmp(elem->name, name) == 0)
4166                         return elem;
4167
4168         return NULL;
4169 }
4170
4171 static struct table_type *
4172 table_type_resolve(struct rte_swx_pipeline *p,
4173                    const char *recommended_type_name,
4174                    enum rte_swx_table_match_type match_type)
4175 {
4176         struct table_type *elem;
4177
4178         /* Only consider the recommended type if the match type is correct. */
4179         if (recommended_type_name)
4180                 TAILQ_FOREACH(elem, &p->table_types, node)
4181                         if (!strcmp(elem->name, recommended_type_name) &&
4182                             (elem->match_type == match_type))
4183                                 return elem;
4184
4185         /* Ignore the recommended type and get the first element with this match
4186          * type.
4187          */
4188         TAILQ_FOREACH(elem, &p->table_types, node)
4189                 if (elem->match_type == match_type)
4190                         return elem;
4191
4192         return NULL;
4193 }
4194
4195 static struct table *
4196 table_find(struct rte_swx_pipeline *p, const char *name)
4197 {
4198         struct table *elem;
4199
4200         TAILQ_FOREACH(elem, &p->tables, node)
4201                 if (strcmp(elem->name, name) == 0)
4202                         return elem;
4203
4204         return NULL;
4205 }
4206
4207 static struct table *
4208 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
4209 {
4210         struct table *table = NULL;
4211
4212         TAILQ_FOREACH(table, &p->tables, node)
4213                 if (table->id == id)
4214                         return table;
4215
4216         return NULL;
4217 }
4218
4219 int
4220 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
4221                                      const char *name,
4222                                      enum rte_swx_table_match_type match_type,
4223                                      struct rte_swx_table_ops *ops)
4224 {
4225         struct table_type *elem;
4226
4227         CHECK(p, EINVAL);
4228
4229         CHECK_NAME(name, EINVAL);
4230         CHECK(!table_type_find(p, name), EEXIST);
4231
4232         CHECK(ops, EINVAL);
4233         CHECK(ops->create, EINVAL);
4234         CHECK(ops->lkp, EINVAL);
4235         CHECK(ops->free, EINVAL);
4236
4237         /* Node allocation. */
4238         elem = calloc(1, sizeof(struct table_type));
4239         CHECK(elem, ENOMEM);
4240
4241         /* Node initialization. */
4242         strcpy(elem->name, name);
4243         elem->match_type = match_type;
4244         memcpy(&elem->ops, ops, sizeof(*ops));
4245
4246         /* Node add to tailq. */
4247         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
4248
4249         return 0;
4250 }
4251
4252 static enum rte_swx_table_match_type
4253 table_match_type_resolve(struct rte_swx_match_field_params *fields,
4254                          uint32_t n_fields)
4255 {
4256         uint32_t i;
4257
4258         for (i = 0; i < n_fields; i++)
4259                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
4260                         break;
4261
4262         if (i == n_fields)
4263                 return RTE_SWX_TABLE_MATCH_EXACT;
4264
4265         if ((i == n_fields - 1) &&
4266             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
4267                 return RTE_SWX_TABLE_MATCH_LPM;
4268
4269         return RTE_SWX_TABLE_MATCH_WILDCARD;
4270 }
4271
4272 int
4273 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
4274                               const char *name,
4275                               struct rte_swx_pipeline_table_params *params,
4276                               const char *recommended_table_type_name,
4277                               const char *args,
4278                               uint32_t size)
4279 {
4280         struct table_type *type;
4281         struct table *t;
4282         struct action *default_action;
4283         struct header *header = NULL;
4284         int is_header = 0;
4285         uint32_t offset_prev = 0, action_data_size_max = 0, i;
4286
4287         CHECK(p, EINVAL);
4288
4289         CHECK_NAME(name, EINVAL);
4290         CHECK(!table_find(p, name), EEXIST);
4291
4292         CHECK(params, EINVAL);
4293
4294         /* Match checks. */
4295         CHECK(!params->n_fields || params->fields, EINVAL);
4296         for (i = 0; i < params->n_fields; i++) {
4297                 struct rte_swx_match_field_params *field = &params->fields[i];
4298                 struct header *h;
4299                 struct field *hf, *mf;
4300                 uint32_t offset;
4301
4302                 CHECK_NAME(field->name, EINVAL);
4303
4304                 hf = header_field_parse(p, field->name, &h);
4305                 mf = metadata_field_parse(p, field->name);
4306                 CHECK(hf || mf, EINVAL);
4307
4308                 offset = hf ? hf->offset : mf->offset;
4309
4310                 if (i == 0) {
4311                         is_header = hf ? 1 : 0;
4312                         header = hf ? h : NULL;
4313                         offset_prev = offset;
4314
4315                         continue;
4316                 }
4317
4318                 CHECK((is_header && hf && (h->id == header->id)) ||
4319                       (!is_header && mf), EINVAL);
4320
4321                 CHECK(offset > offset_prev, EINVAL);
4322                 offset_prev = offset;
4323         }
4324
4325         /* Action checks. */
4326         CHECK(params->n_actions, EINVAL);
4327         CHECK(params->action_names, EINVAL);
4328         for (i = 0; i < params->n_actions; i++) {
4329                 const char *action_name = params->action_names[i];
4330                 struct action *a;
4331                 uint32_t action_data_size;
4332
4333                 CHECK(action_name, EINVAL);
4334
4335                 a = action_find(p, action_name);
4336                 CHECK(a, EINVAL);
4337
4338                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
4339                 if (action_data_size > action_data_size_max)
4340                         action_data_size_max = action_data_size;
4341         }
4342
4343         CHECK(params->default_action_name, EINVAL);
4344         for (i = 0; i < p->n_actions; i++)
4345                 if (!strcmp(params->action_names[i],
4346                             params->default_action_name))
4347                         break;
4348         CHECK(i < params->n_actions, EINVAL);
4349         default_action = action_find(p, params->default_action_name);
4350         CHECK((default_action->st && params->default_action_data) ||
4351               !params->default_action_data, EINVAL);
4352
4353         /* Table type checks. */
4354         if (params->n_fields) {
4355                 enum rte_swx_table_match_type match_type;
4356
4357                 match_type = table_match_type_resolve(params->fields,
4358                                                       params->n_fields);
4359                 type = table_type_resolve(p,
4360                                           recommended_table_type_name,
4361                                           match_type);
4362                 CHECK(type, EINVAL);
4363         } else {
4364                 type = NULL;
4365         }
4366
4367         /* Memory allocation. */
4368         t = calloc(1, sizeof(struct table));
4369         CHECK(t, ENOMEM);
4370
4371         t->fields = calloc(params->n_fields, sizeof(struct match_field));
4372         if (!t->fields) {
4373                 free(t);
4374                 CHECK(0, ENOMEM);
4375         }
4376
4377         t->actions = calloc(params->n_actions, sizeof(struct action *));
4378         if (!t->actions) {
4379                 free(t->fields);
4380                 free(t);
4381                 CHECK(0, ENOMEM);
4382         }
4383
4384         if (action_data_size_max) {
4385                 t->default_action_data = calloc(1, action_data_size_max);
4386                 if (!t->default_action_data) {
4387                         free(t->actions);
4388                         free(t->fields);
4389                         free(t);
4390                         CHECK(0, ENOMEM);
4391                 }
4392         }
4393
4394         /* Node initialization. */
4395         strcpy(t->name, name);
4396         if (args && args[0])
4397                 strcpy(t->args, args);
4398         t->type = type;
4399
4400         for (i = 0; i < params->n_fields; i++) {
4401                 struct rte_swx_match_field_params *field = &params->fields[i];
4402                 struct match_field *f = &t->fields[i];
4403
4404                 f->match_type = field->match_type;
4405                 f->field = is_header ?
4406                         header_field_parse(p, field->name, NULL) :
4407                         metadata_field_parse(p, field->name);
4408         }
4409         t->n_fields = params->n_fields;
4410         t->is_header = is_header;
4411         t->header = header;
4412
4413         for (i = 0; i < params->n_actions; i++)
4414                 t->actions[i] = action_find(p, params->action_names[i]);
4415         t->default_action = default_action;
4416         if (default_action->st)
4417                 memcpy(t->default_action_data,
4418                        params->default_action_data,
4419                        default_action->st->n_bits / 8);
4420         t->n_actions = params->n_actions;
4421         t->default_action_is_const = params->default_action_is_const;
4422         t->action_data_size_max = action_data_size_max;
4423
4424         t->size = size;
4425         t->id = p->n_tables;
4426
4427         /* Node add to tailq. */
4428         TAILQ_INSERT_TAIL(&p->tables, t, node);
4429         p->n_tables++;
4430
4431         return 0;
4432 }
4433
4434 static struct rte_swx_table_params *
4435 table_params_get(struct table *table)
4436 {
4437         struct rte_swx_table_params *params;
4438         struct field *first, *last;
4439         uint8_t *key_mask;
4440         uint32_t key_size, key_offset, action_data_size, i;
4441
4442         /* Memory allocation. */
4443         params = calloc(1, sizeof(struct rte_swx_table_params));
4444         if (!params)
4445                 return NULL;
4446
4447         /* Key offset and size. */
4448         first = table->fields[0].field;
4449         last = table->fields[table->n_fields - 1].field;
4450         key_offset = first->offset / 8;
4451         key_size = (last->offset + last->n_bits - first->offset) / 8;
4452
4453         /* Memory allocation. */
4454         key_mask = calloc(1, key_size);
4455         if (!key_mask) {
4456                 free(params);
4457                 return NULL;
4458         }
4459
4460         /* Key mask. */
4461         for (i = 0; i < table->n_fields; i++) {
4462                 struct field *f = table->fields[i].field;
4463                 uint32_t start = (f->offset - first->offset) / 8;
4464                 size_t size = f->n_bits / 8;
4465
4466                 memset(&key_mask[start], 0xFF, size);
4467         }
4468
4469         /* Action data size. */
4470         action_data_size = 0;
4471         for (i = 0; i < table->n_actions; i++) {
4472                 struct action *action = table->actions[i];
4473                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
4474
4475                 if (ads > action_data_size)
4476                         action_data_size = ads;
4477         }
4478
4479         /* Fill in. */
4480         params->match_type = table->type->match_type;
4481         params->key_size = key_size;
4482         params->key_offset = key_offset;
4483         params->key_mask0 = key_mask;
4484         params->action_data_size = action_data_size;
4485         params->n_keys_max = table->size;
4486
4487         return params;
4488 }
4489
4490 static void
4491 table_params_free(struct rte_swx_table_params *params)
4492 {
4493         if (!params)
4494                 return;
4495
4496         free(params->key_mask0);
4497         free(params);
4498 }
4499
4500 static int
4501 table_state_build(struct rte_swx_pipeline *p)
4502 {
4503         struct table *table;
4504
4505         p->table_state = calloc(p->n_tables,
4506                                 sizeof(struct rte_swx_table_state));
4507         CHECK(p->table_state, ENOMEM);
4508
4509         TAILQ_FOREACH(table, &p->tables, node) {
4510                 struct rte_swx_table_state *ts = &p->table_state[table->id];
4511
4512                 if (table->type) {
4513                         struct rte_swx_table_params *params;
4514
4515                         /* ts->obj. */
4516                         params = table_params_get(table);
4517                         CHECK(params, ENOMEM);
4518
4519                         ts->obj = table->type->ops.create(params,
4520                                 NULL,
4521                                 table->args,
4522                                 p->numa_node);
4523
4524                         table_params_free(params);
4525                         CHECK(ts->obj, ENODEV);
4526                 }
4527
4528                 /* ts->default_action_data. */
4529                 if (table->action_data_size_max) {
4530                         ts->default_action_data =
4531                                 malloc(table->action_data_size_max);
4532                         CHECK(ts->default_action_data, ENOMEM);
4533
4534                         memcpy(ts->default_action_data,
4535                                table->default_action_data,
4536                                table->action_data_size_max);
4537                 }
4538
4539                 /* ts->default_action_id. */
4540                 ts->default_action_id = table->default_action->id;
4541         }
4542
4543         return 0;
4544 }
4545
4546 static void
4547 table_state_build_free(struct rte_swx_pipeline *p)
4548 {
4549         uint32_t i;
4550
4551         if (!p->table_state)
4552                 return;
4553
4554         for (i = 0; i < p->n_tables; i++) {
4555                 struct rte_swx_table_state *ts = &p->table_state[i];
4556                 struct table *table = table_find_by_id(p, i);
4557
4558                 /* ts->obj. */
4559                 if (table->type && ts->obj)
4560                         table->type->ops.free(ts->obj);
4561
4562                 /* ts->default_action_data. */
4563                 free(ts->default_action_data);
4564         }
4565
4566         free(p->table_state);
4567         p->table_state = NULL;
4568 }
4569
4570 static void
4571 table_state_free(struct rte_swx_pipeline *p)
4572 {
4573         table_state_build_free(p);
4574 }
4575
4576 static int
4577 table_stub_lkp(void *table __rte_unused,
4578                void *mailbox __rte_unused,
4579                uint8_t **key __rte_unused,
4580                uint64_t *action_id __rte_unused,
4581                uint8_t **action_data __rte_unused,
4582                int *hit)
4583 {
4584         *hit = 0;
4585         return 1; /* DONE. */
4586 }
4587
4588 static int
4589 table_build(struct rte_swx_pipeline *p)
4590 {
4591         uint32_t i;
4592
4593         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
4594                 struct thread *t = &p->threads[i];
4595                 struct table *table;
4596
4597                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
4598                 CHECK(t->tables, ENOMEM);
4599
4600                 TAILQ_FOREACH(table, &p->tables, node) {
4601                         struct table_runtime *r = &t->tables[table->id];
4602
4603                         if (table->type) {
4604                                 uint64_t size;
4605
4606                                 size = table->type->ops.mailbox_size_get();
4607
4608                                 /* r->func. */
4609                                 r->func = table->type->ops.lkp;
4610
4611                                 /* r->mailbox. */
4612                                 if (size) {
4613                                         r->mailbox = calloc(1, size);
4614                                         CHECK(r->mailbox, ENOMEM);
4615                                 }
4616
4617                                 /* r->key. */
4618                                 r->key = table->is_header ?
4619                                         &t->structs[table->header->struct_id] :
4620                                         &t->structs[p->metadata_struct_id];
4621                         } else {
4622                                 r->func = table_stub_lkp;
4623                         }
4624                 }
4625         }
4626
4627         return 0;
4628 }
4629
4630 static void
4631 table_build_free(struct rte_swx_pipeline *p)
4632 {
4633         uint32_t i;
4634
4635         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
4636                 struct thread *t = &p->threads[i];
4637                 uint32_t j;
4638
4639                 if (!t->tables)
4640                         continue;
4641
4642                 for (j = 0; j < p->n_tables; j++) {
4643                         struct table_runtime *r = &t->tables[j];
4644
4645                         free(r->mailbox);
4646                 }
4647
4648                 free(t->tables);
4649                 t->tables = NULL;
4650         }
4651 }
4652
4653 static void
4654 table_free(struct rte_swx_pipeline *p)
4655 {
4656         table_build_free(p);
4657
4658         /* Tables. */
4659         for ( ; ; ) {
4660                 struct table *elem;
4661
4662                 elem = TAILQ_FIRST(&p->tables);
4663                 if (!elem)
4664                         break;
4665
4666                 TAILQ_REMOVE(&p->tables, elem, node);
4667                 free(elem->fields);
4668                 free(elem->actions);
4669                 free(elem->default_action_data);
4670                 free(elem);
4671         }
4672
4673         /* Table types. */
4674         for ( ; ; ) {
4675                 struct table_type *elem;
4676
4677                 elem = TAILQ_FIRST(&p->table_types);
4678                 if (!elem)
4679                         break;
4680
4681                 TAILQ_REMOVE(&p->table_types, elem, node);
4682                 free(elem);
4683         }
4684 }
4685
4686 /*
4687  * Pipeline.
4688  */
4689 int
4690 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
4691 {
4692         struct rte_swx_pipeline *pipeline;
4693
4694         /* Check input parameters. */
4695         CHECK(p, EINVAL);
4696
4697         /* Memory allocation. */
4698         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
4699         CHECK(pipeline, ENOMEM);
4700
4701         /* Initialization. */
4702         TAILQ_INIT(&pipeline->struct_types);
4703         TAILQ_INIT(&pipeline->port_in_types);
4704         TAILQ_INIT(&pipeline->ports_in);
4705         TAILQ_INIT(&pipeline->port_out_types);
4706         TAILQ_INIT(&pipeline->ports_out);
4707         TAILQ_INIT(&pipeline->extern_types);
4708         TAILQ_INIT(&pipeline->extern_objs);
4709         TAILQ_INIT(&pipeline->extern_funcs);
4710         TAILQ_INIT(&pipeline->headers);
4711         TAILQ_INIT(&pipeline->actions);
4712         TAILQ_INIT(&pipeline->table_types);
4713         TAILQ_INIT(&pipeline->tables);
4714
4715         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
4716         pipeline->numa_node = numa_node;
4717
4718         *p = pipeline;
4719         return 0;
4720 }
4721
4722 void
4723 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
4724 {
4725         if (!p)
4726                 return;
4727
4728         free(p->instructions);
4729
4730         table_state_free(p);
4731         table_free(p);
4732         action_free(p);
4733         metadata_free(p);
4734         header_free(p);
4735         extern_func_free(p);
4736         extern_obj_free(p);
4737         port_out_free(p);
4738         port_in_free(p);
4739         struct_free(p);
4740
4741         free(p);
4742 }
4743
4744 int
4745 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
4746                                      const char **instructions,
4747                                      uint32_t n_instructions)
4748 {
4749         int err;
4750         uint32_t i;
4751
4752         err = instruction_config(p, NULL, instructions, n_instructions);
4753         if (err)
4754                 return err;
4755
4756         /* Thread instruction pointer reset. */
4757         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
4758                 struct thread *t = &p->threads[i];
4759
4760                 thread_ip_reset(p, t);
4761         }
4762
4763         return 0;
4764 }
4765
4766 int
4767 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
4768 {
4769         int status;
4770
4771         CHECK(p, EINVAL);
4772         CHECK(p->build_done == 0, EEXIST);
4773
4774         status = port_in_build(p);
4775         if (status)
4776                 goto error;
4777
4778         status = port_out_build(p);
4779         if (status)
4780                 goto error;
4781
4782         status = struct_build(p);
4783         if (status)
4784                 goto error;
4785
4786         status = extern_obj_build(p);
4787         if (status)
4788                 goto error;
4789
4790         status = extern_func_build(p);
4791         if (status)
4792                 goto error;
4793
4794         status = header_build(p);
4795         if (status)
4796                 goto error;
4797
4798         status = metadata_build(p);
4799         if (status)
4800                 goto error;
4801
4802         status = action_build(p);
4803         if (status)
4804                 goto error;
4805
4806         status = table_build(p);
4807         if (status)
4808                 goto error;
4809
4810         status = table_state_build(p);
4811         if (status)
4812                 goto error;
4813
4814         p->build_done = 1;
4815         return 0;
4816
4817 error:
4818         table_state_build_free(p);
4819         table_build_free(p);
4820         action_build_free(p);
4821         metadata_build_free(p);
4822         header_build_free(p);
4823         extern_func_build_free(p);
4824         extern_obj_build_free(p);
4825         port_out_build_free(p);
4826         port_in_build_free(p);
4827         struct_build_free(p);
4828
4829         return status;
4830 }
4831
4832 void
4833 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
4834 {
4835         uint32_t i;
4836
4837         for (i = 0; i < n_instructions; i++)
4838                 instr_exec(p);
4839 }
4840
4841 /*
4842  * Control.
4843  */
4844 int
4845 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
4846                                  struct rte_swx_table_state **table_state)
4847 {
4848         if (!p || !table_state || !p->build_done)
4849                 return -EINVAL;
4850
4851         *table_state = p->table_state;
4852         return 0;
4853 }
4854
4855 int
4856 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
4857                                  struct rte_swx_table_state *table_state)
4858 {
4859         if (!p || !table_state || !p->build_done)
4860                 return -EINVAL;
4861
4862         p->table_state = table_state;
4863         return 0;
4864 }