lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9 #include <arpa/inet.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_prefetch.h>
  13 #include <rte_byteorder.h>
  14
  15 #include "rte_swx_pipeline.h"
  16 #include "rte_swx_ctl.h"
  17
  18 #define CHECK(condition, err_code)                                             \
  19 do {                                                                           \
  20         if (!(condition))                                                      \
  21                 return -(err_code);                                            \
  22 } while (0)
  23
  24 #define CHECK_NAME(name, err_code)                                             \
  25         CHECK((name) && (name)[0], err_code)
  26
  27 #ifndef TRACE_LEVEL
  28 #define TRACE_LEVEL 0
  29 #endif
  30
  31 #if TRACE_LEVEL
  32 #define TRACE(...) printf(__VA_ARGS__)
  33 #else
  34 #define TRACE(...)
  35 #endif
  36
  37 #define ntoh64(x) rte_be_to_cpu_64(x)
  38 #define hton64(x) rte_cpu_to_be_64(x)
  39
  40 /*
  41  * Struct.
  42  */
  43 struct field {
  44         char name[RTE_SWX_NAME_SIZE];
  45         uint32_t n_bits;
  46         uint32_t offset;
  47 };
  48
  49 struct struct_type {
  50         TAILQ_ENTRY(struct_type) node;
  51         char name[RTE_SWX_NAME_SIZE];
  52         struct field *fields;
  53         uint32_t n_fields;
  54         uint32_t n_bits;
  55 };
  56
  57 TAILQ_HEAD(struct_type_tailq, struct_type);
  58
  59 /*
  60  * Input port.
  61  */
  62 struct port_in_type {
  63         TAILQ_ENTRY(port_in_type) node;
  64         char name[RTE_SWX_NAME_SIZE];
  65         struct rte_swx_port_in_ops ops;
  66 };
  67
  68 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  69
  70 struct port_in {
  71         TAILQ_ENTRY(port_in) node;
  72         struct port_in_type *type;
  73         void *obj;
  74         uint32_t id;
  75 };
  76
  77 TAILQ_HEAD(port_in_tailq, port_in);
  78
  79 struct port_in_runtime {
  80         rte_swx_port_in_pkt_rx_t pkt_rx;
  81         void *obj;
  82 };
  83
  84 /*
  85  * Output port.
  86  */
  87 struct port_out_type {
  88         TAILQ_ENTRY(port_out_type) node;
  89         char name[RTE_SWX_NAME_SIZE];
  90         struct rte_swx_port_out_ops ops;
  91 };
  92
  93 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  94
  95 struct port_out {
  96         TAILQ_ENTRY(port_out) node;
  97         struct port_out_type *type;
  98         void *obj;
  99         uint32_t id;
 100 };
 101
 102 TAILQ_HEAD(port_out_tailq, port_out);
 103
 104 struct port_out_runtime {
 105         rte_swx_port_out_pkt_tx_t pkt_tx;
 106         rte_swx_port_out_flush_t flush;
 107         void *obj;
 108 };
 109
 110 /*
 111  * Extern object.
 112  */
 113 struct extern_type_member_func {
 114         TAILQ_ENTRY(extern_type_member_func) node;
 115         char name[RTE_SWX_NAME_SIZE];
 116         rte_swx_extern_type_member_func_t func;
 117         uint32_t id;
 118 };
 119
 120 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 121
 122 struct extern_type {
 123         TAILQ_ENTRY(extern_type) node;
 124         char name[RTE_SWX_NAME_SIZE];
 125         struct struct_type *mailbox_struct_type;
 126         rte_swx_extern_type_constructor_t constructor;
 127         rte_swx_extern_type_destructor_t destructor;
 128         struct extern_type_member_func_tailq funcs;
 129         uint32_t n_funcs;
 130 };
 131
 132 TAILQ_HEAD(extern_type_tailq, extern_type);
 133
 134 struct extern_obj {
 135         TAILQ_ENTRY(extern_obj) node;
 136         char name[RTE_SWX_NAME_SIZE];
 137         struct extern_type *type;
 138         void *obj;
 139         uint32_t struct_id;
 140         uint32_t id;
 141 };
 142
 143 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 144
 145 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 146 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 147 #endif
 148
 149 struct extern_obj_runtime {
 150         void *obj;
 151         uint8_t *mailbox;
 152         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 153 };
 154
 155 /*
 156  * Extern function.
 157  */
 158 struct extern_func {
 159         TAILQ_ENTRY(extern_func) node;
 160         char name[RTE_SWX_NAME_SIZE];
 161         struct struct_type *mailbox_struct_type;
 162         rte_swx_extern_func_t func;
 163         uint32_t struct_id;
 164         uint32_t id;
 165 };
 166
 167 TAILQ_HEAD(extern_func_tailq, extern_func);
 168
 169 struct extern_func_runtime {
 170         uint8_t *mailbox;
 171         rte_swx_extern_func_t func;
 172 };
 173
 174 /*
 175  * Header.
 176  */
 177 struct header {
 178         TAILQ_ENTRY(header) node;
 179         char name[RTE_SWX_NAME_SIZE];
 180         struct struct_type *st;
 181         uint32_t struct_id;
 182         uint32_t id;
 183 };
 184
 185 TAILQ_HEAD(header_tailq, header);
 186
 187 struct header_runtime {
 188         uint8_t *ptr0;
 189 };
 190
 191 struct header_out_runtime {
 192         uint8_t *ptr0;
 193         uint8_t *ptr;
 194         uint32_t n_bytes;
 195 };
 196
 197 /*
 198  * Instruction.
 199  */
 200
 201 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 202  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 203  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 204  * when transferred to packet meta-data and in NBO when transferred to packet
 205  * headers.
 206  */
 207
 208 /* Notation conventions:
 209  *    -Header field: H = h.header.field (dst/src)
 210  *    -Meta-data field: M = m.field (dst/src)
 211  *    -Extern object mailbox field: E = e.field (dst/src)
 212  *    -Extern function mailbox field: F = f.field (dst/src)
 213  *    -Table action data field: T = t.field (src only)
 214  *    -Immediate value: I = 32-bit unsigned value (src only)
 215  */
 216
 217 enum instruction_type {
 218         /* rx m.port_in */
 219         INSTR_RX,
 220
 221         /* tx m.port_out */
 222         INSTR_TX,
 223
 224         /* extract h.header */
 225         INSTR_HDR_EXTRACT,
 226         INSTR_HDR_EXTRACT2,
 227         INSTR_HDR_EXTRACT3,
 228         INSTR_HDR_EXTRACT4,
 229         INSTR_HDR_EXTRACT5,
 230         INSTR_HDR_EXTRACT6,
 231         INSTR_HDR_EXTRACT7,
 232         INSTR_HDR_EXTRACT8,
 233
 234         /* emit h.header */
 235         INSTR_HDR_EMIT,
 236         INSTR_HDR_EMIT_TX,
 237         INSTR_HDR_EMIT2_TX,
 238         INSTR_HDR_EMIT3_TX,
 239         INSTR_HDR_EMIT4_TX,
 240         INSTR_HDR_EMIT5_TX,
 241         INSTR_HDR_EMIT6_TX,
 242         INSTR_HDR_EMIT7_TX,
 243         INSTR_HDR_EMIT8_TX,
 244
 245         /* validate h.header */
 246         INSTR_HDR_VALIDATE,
 247
 248         /* invalidate h.header */
 249         INSTR_HDR_INVALIDATE,
 250
 251         /* mov dst src
 252          * dst = src
 253          * dst = HMEF, src = HMEFTI
 254          */
 255         INSTR_MOV,   /* dst = MEF, src = MEFT */
 256         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 257         INSTR_MOV_I, /* dst = HMEF, src = I */
 258
 259         /* dma h.header t.field
 260          * memcpy(h.header, t.field, sizeof(h.header))
 261          */
 262         INSTR_DMA_HT,
 263         INSTR_DMA_HT2,
 264         INSTR_DMA_HT3,
 265         INSTR_DMA_HT4,
 266         INSTR_DMA_HT5,
 267         INSTR_DMA_HT6,
 268         INSTR_DMA_HT7,
 269         INSTR_DMA_HT8,
 270
 271         /* add dst src
 272          * dst += src
 273          * dst = HMEF, src = HMEFTI
 274          */
 275         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 276         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 277         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 278         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 279         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 280         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 281
 282         /* sub dst src
 283          * dst -= src
 284          * dst = HMEF, src = HMEFTI
 285          */
 286         INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
 287         INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
 288         INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
 289         INSTR_ALU_SUB_HH, /* dst = H, src = H */
 290         INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
 291         INSTR_ALU_SUB_HI, /* dst = H, src = I */
 292
 293         /* ckadd dst src
 294          * dst = dst '+ src[0:1] '+ src[2:3] + ...
 295          * dst = H, src = {H, h.header}
 296          */
 297         INSTR_ALU_CKADD_FIELD,    /* src = H */
 298         INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
 299         INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */
 300
 301         /* cksub dst src
 302          * dst = dst '- src
 303          * dst = H, src = H
 304          */
 305         INSTR_ALU_CKSUB_FIELD,
 306
 307         /* and dst src
 308          * dst &= src
 309          * dst = HMEF, src = HMEFTI
 310          */
 311         INSTR_ALU_AND,   /* dst = MEF, src = MEFT */
 312         INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 313         INSTR_ALU_AND_I, /* dst = HMEF, src = I */
 314
 315         /* or dst src
 316          * dst |= src
 317          * dst = HMEF, src = HMEFTI
 318          */
 319         INSTR_ALU_OR,   /* dst = MEF, src = MEFT */
 320         INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 321         INSTR_ALU_OR_I, /* dst = HMEF, src = I */
 322
 323         /* xor dst src
 324          * dst ^= src
 325          * dst = HMEF, src = HMEFTI
 326          */
 327         INSTR_ALU_XOR,   /* dst = MEF, src = MEFT */
 328         INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 329         INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
 330
 331         /* shl dst src
 332          * dst <<= src
 333          * dst = HMEF, src = HMEFTI
 334          */
 335         INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
 336         INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
 337         INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
 338         INSTR_ALU_SHL_HH, /* dst = H, src = H */
 339         INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
 340         INSTR_ALU_SHL_HI, /* dst = H, src = I */
 341
 342         /* shr dst src
 343          * dst >>= src
 344          * dst = HMEF, src = HMEFTI
 345          */
 346         INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
 347         INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
 348         INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
 349         INSTR_ALU_SHR_HH, /* dst = H, src = H */
 350         INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
 351         INSTR_ALU_SHR_HI, /* dst = H, src = I */
 352
 353         /* table TABLE */
 354         INSTR_TABLE,
 355
 356         /* extern e.obj.func */
 357         INSTR_EXTERN_OBJ,
 358
 359         /* extern f.func */
 360         INSTR_EXTERN_FUNC,
 361 };
 362
 363 struct instr_operand {
 364         uint8_t struct_id;
 365         uint8_t n_bits;
 366         uint8_t offset;
 367         uint8_t pad;
 368 };
 369
 370 struct instr_io {
 371         struct {
 372                 uint8_t offset;
 373                 uint8_t n_bits;
 374                 uint8_t pad[2];
 375         } io;
 376
 377         struct {
 378                 uint8_t header_id[8];
 379                 uint8_t struct_id[8];
 380                 uint8_t n_bytes[8];
 381         } hdr;
 382 };
 383
 384 struct instr_hdr_validity {
 385         uint8_t header_id;
 386 };
 387
 388 struct instr_table {
 389         uint8_t table_id;
 390 };
 391
 392 struct instr_extern_obj {
 393         uint8_t ext_obj_id;
 394         uint8_t func_id;
 395 };
 396
 397 struct instr_extern_func {
 398         uint8_t ext_func_id;
 399 };
 400
 401 struct instr_dst_src {
 402         struct instr_operand dst;
 403         union {
 404                 struct instr_operand src;
 405                 uint32_t src_val;
 406         };
 407 };
 408
 409 struct instr_dma {
 410         struct {
 411                 uint8_t header_id[8];
 412                 uint8_t struct_id[8];
 413         } dst;
 414
 415         struct {
 416                 uint8_t offset[8];
 417         } src;
 418
 419         uint16_t n_bytes[8];
 420 };
 421
 422 struct instruction {
 423         enum instruction_type type;
 424         union {
 425                 struct instr_io io;
 426                 struct instr_hdr_validity valid;
 427                 struct instr_dst_src mov;
 428                 struct instr_dma dma;
 429                 struct instr_dst_src alu;
 430                 struct instr_table table;
 431                 struct instr_extern_obj ext_obj;
 432                 struct instr_extern_func ext_func;
 433         };
 434 };
 435
 436 struct instruction_data {
 437         char label[RTE_SWX_NAME_SIZE];
 438         char jmp_label[RTE_SWX_NAME_SIZE];
 439         uint32_t n_users; /* user = jmp instruction to this instruction. */
 440         int invalid;
 441 };
 442
 443 /*
 444  * Action.
 445  */
 446 struct action {
 447         TAILQ_ENTRY(action) node;
 448         char name[RTE_SWX_NAME_SIZE];
 449         struct struct_type *st;
 450         struct instruction *instructions;
 451         uint32_t n_instructions;
 452         uint32_t id;
 453 };
 454
 455 TAILQ_HEAD(action_tailq, action);
 456
 457 /*
 458  * Table.
 459  */
 460 struct table_type {
 461         TAILQ_ENTRY(table_type) node;
 462         char name[RTE_SWX_NAME_SIZE];
 463         enum rte_swx_table_match_type match_type;
 464         struct rte_swx_table_ops ops;
 465 };
 466
 467 TAILQ_HEAD(table_type_tailq, table_type);
 468
 469 struct match_field {
 470         enum rte_swx_table_match_type match_type;
 471         struct field *field;
 472 };
 473
 474 struct table {
 475         TAILQ_ENTRY(table) node;
 476         char name[RTE_SWX_NAME_SIZE];
 477         char args[RTE_SWX_NAME_SIZE];
 478         struct table_type *type; /* NULL when n_fields == 0. */
 479
 480         /* Match. */
 481         struct match_field *fields;
 482         uint32_t n_fields;
 483         int is_header; /* Only valid when n_fields > 0. */
 484         struct header *header; /* Only valid when n_fields > 0. */
 485
 486         /* Action. */
 487         struct action **actions;
 488         struct action *default_action;
 489         uint8_t *default_action_data;
 490         uint32_t n_actions;
 491         int default_action_is_const;
 492         uint32_t action_data_size_max;
 493
 494         uint32_t size;
 495         uint32_t id;
 496 };
 497
 498 TAILQ_HEAD(table_tailq, table);
 499
 500 struct table_runtime {
 501         rte_swx_table_lookup_t func;
 502         void *mailbox;
 503         uint8_t **key;
 504 };
 505
 506 /*
 507  * Pipeline.
 508  */
 509 struct thread {
 510         /* Packet. */
 511         struct rte_swx_pkt pkt;
 512         uint8_t *ptr;
 513
 514         /* Structures. */
 515         uint8_t **structs;
 516
 517         /* Packet headers. */
 518         struct header_runtime *headers; /* Extracted or generated headers. */
 519         struct header_out_runtime *headers_out; /* Emitted headers. */
 520         uint8_t *header_storage;
 521         uint8_t *header_out_storage;
 522         uint64_t valid_headers;
 523         uint32_t n_headers_out;
 524
 525         /* Packet meta-data. */
 526         uint8_t *metadata;
 527
 528         /* Tables. */
 529         struct table_runtime *tables;
 530         struct rte_swx_table_state *table_state;
 531         uint64_t action_id;
 532         int hit; /* 0 = Miss, 1 = Hit. */
 533
 534         /* Extern objects and functions. */
 535         struct extern_obj_runtime *extern_objs;
 536         struct extern_func_runtime *extern_funcs;
 537
 538         /* Instructions. */
 539         struct instruction *ip;
 540         struct instruction *ret;
 541 };
 542
 543 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 544 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 545 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 546
 547 #define ALU(thread, ip, operator)  \
 548 {                                                                              \
 549         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 550         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 551         uint64_t dst64 = *dst64_ptr;                                           \
 552         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 553         uint64_t dst = dst64 & dst64_mask;                                     \
 554                                                                                \
 555         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 556         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 557         uint64_t src64 = *src64_ptr;                                           \
 558         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 559         uint64_t src = src64 & src64_mask;                                     \
 560                                                                                \
 561         uint64_t result = dst operator src;                                    \
 562                                                                                \
 563         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 564 }
 565
 566 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 567
 568 #define ALU_S(thread, ip, operator)  \
 569 {                                                                              \
 570         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 571         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 572         uint64_t dst64 = *dst64_ptr;                                           \
 573         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 574         uint64_t dst = dst64 & dst64_mask;                                     \
 575                                                                                \
 576         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 577         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 578         uint64_t src64 = *src64_ptr;                                           \
 579         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 580                                                                                \
 581         uint64_t result = dst operator src;                                    \
 582                                                                                \
 583         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 584 }
 585
 586 #define ALU_MH ALU_S
 587
 588 #define ALU_HM(thread, ip, operator)  \
 589 {                                                                              \
 590         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 591         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 592         uint64_t dst64 = *dst64_ptr;                                           \
 593         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 594         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 595                                                                                \
 596         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 597         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 598         uint64_t src64 = *src64_ptr;                                           \
 599         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 600         uint64_t src = src64 & src64_mask;                                     \
 601                                                                                \
 602         uint64_t result = dst operator src;                                    \
 603         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 604                                                                                \
 605         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 606 }
 607
 608 #define ALU_HH(thread, ip, operator)  \
 609 {                                                                              \
 610         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 611         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 612         uint64_t dst64 = *dst64_ptr;                                           \
 613         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 614         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 615                                                                                \
 616         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 617         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 618         uint64_t src64 = *src64_ptr;                                           \
 619         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 620                                                                                \
 621         uint64_t result = dst operator src;                                    \
 622         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 623                                                                                \
 624         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 625 }
 626
 627 #else
 628
 629 #define ALU_S ALU
 630 #define ALU_MH ALU
 631 #define ALU_HM ALU
 632 #define ALU_HH ALU
 633
 634 #endif
 635
 636 #define ALU_I(thread, ip, operator)  \
 637 {                                                                              \
 638         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 639         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 640         uint64_t dst64 = *dst64_ptr;                                           \
 641         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 642         uint64_t dst = dst64 & dst64_mask;                                     \
 643                                                                                \
 644         uint64_t src = (ip)->alu.src_val;                                      \
 645                                                                                \
 646         uint64_t result = dst operator src;                                    \
 647                                                                                \
 648         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 649 }
 650
 651 #define ALU_MI ALU_I
 652
 653 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 654
 655 #define ALU_HI(thread, ip, operator)  \
 656 {                                                                              \
 657         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 658         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 659         uint64_t dst64 = *dst64_ptr;                                           \
 660         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 661         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 662                                                                                \
 663         uint64_t src = (ip)->alu.src_val;                                      \
 664                                                                                \
 665         uint64_t result = dst operator src;                                    \
 666         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 667                                                                                \
 668         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 669 }
 670
 671 #else
 672
 673 #define ALU_HI ALU_I
 674
 675 #endif
 676
 677 #define MOV(thread, ip)  \
 678 {                                                                              \
 679         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 680         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 681         uint64_t dst64 = *dst64_ptr;                                           \
 682         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 683                                                                                \
 684         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 685         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 686         uint64_t src64 = *src64_ptr;                                           \
 687         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
 688         uint64_t src = src64 & src64_mask;                                     \
 689                                                                                \
 690         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 691 }
 692
 693 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 694
 695 #define MOV_S(thread, ip)  \
 696 {                                                                              \
 697         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 698         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 699         uint64_t dst64 = *dst64_ptr;                                           \
 700         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 701                                                                                \
 702         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 703         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 704         uint64_t src64 = *src64_ptr;                                           \
 705         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
 706                                                                                \
 707         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 708 }
 709
 710 #else
 711
 712 #define MOV_S MOV
 713
 714 #endif
 715
 716 #define MOV_I(thread, ip)  \
 717 {                                                                              \
 718         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 719         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 720         uint64_t dst64 = *dst64_ptr;                                           \
 721         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 722                                                                                \
 723         uint64_t src = (ip)->mov.src_val;                                      \
 724                                                                                \
 725         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 726 }
 727
 728 #define METADATA_READ(thread, offset, n_bits)                                  \
 729 ({                                                                             \
 730         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 731         uint64_t m64 = *m64_ptr;                                               \
 732         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 733         (m64 & m64_mask);                                                      \
 734 })
 735
 736 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
 737 {                                                                              \
 738         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 739         uint64_t m64 = *m64_ptr;                                               \
 740         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 741                                                                                \
 742         uint64_t m_new = value;                                                \
 743                                                                                \
 744         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
 745 }
 746
 747 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 748 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 749 #endif
 750
 751 struct rte_swx_pipeline {
 752         struct struct_type_tailq struct_types;
 753         struct port_in_type_tailq port_in_types;
 754         struct port_in_tailq ports_in;
 755         struct port_out_type_tailq port_out_types;
 756         struct port_out_tailq ports_out;
 757         struct extern_type_tailq extern_types;
 758         struct extern_obj_tailq extern_objs;
 759         struct extern_func_tailq extern_funcs;
 760         struct header_tailq headers;
 761         struct struct_type *metadata_st;
 762         uint32_t metadata_struct_id;
 763         struct action_tailq actions;
 764         struct table_type_tailq table_types;
 765         struct table_tailq tables;
 766
 767         struct port_in_runtime *in;
 768         struct port_out_runtime *out;
 769         struct instruction **action_instructions;
 770         struct rte_swx_table_state *table_state;
 771         struct instruction *instructions;
 772         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 773
 774         uint32_t n_structs;
 775         uint32_t n_ports_in;
 776         uint32_t n_ports_out;
 777         uint32_t n_extern_objs;
 778         uint32_t n_extern_funcs;
 779         uint32_t n_actions;
 780         uint32_t n_tables;
 781         uint32_t n_headers;
 782         uint32_t thread_id;
 783         uint32_t port_id;
 784         uint32_t n_instructions;
 785         int build_done;
 786         int numa_node;
 787 };
 788
 789 /*
 790  * Struct.
 791  */
 792 static struct struct_type *
 793 struct_type_find(struct rte_swx_pipeline *p, const char *name)
 794 {
 795         struct struct_type *elem;
 796
 797         TAILQ_FOREACH(elem, &p->struct_types, node)
 798                 if (strcmp(elem->name, name) == 0)
 799                         return elem;
 800
 801         return NULL;
 802 }
 803
 804 static struct field *
 805 struct_type_field_find(struct struct_type *st, const char *name)
 806 {
 807         uint32_t i;
 808
 809         for (i = 0; i < st->n_fields; i++) {
 810                 struct field *f = &st->fields[i];
 811
 812                 if (strcmp(f->name, name) == 0)
 813                         return f;
 814         }
 815
 816         return NULL;
 817 }
 818
 819 int
 820 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 821                                       const char *name,
 822                                       struct rte_swx_field_params *fields,
 823                                       uint32_t n_fields)
 824 {
 825         struct struct_type *st;
 826         uint32_t i;
 827
 828         CHECK(p, EINVAL);
 829         CHECK_NAME(name, EINVAL);
 830         CHECK(fields, EINVAL);
 831         CHECK(n_fields, EINVAL);
 832
 833         for (i = 0; i < n_fields; i++) {
 834                 struct rte_swx_field_params *f = &fields[i];
 835                 uint32_t j;
 836
 837                 CHECK_NAME(f->name, EINVAL);
 838                 CHECK(f->n_bits, EINVAL);
 839                 CHECK(f->n_bits <= 64, EINVAL);
 840                 CHECK((f->n_bits & 7) == 0, EINVAL);
 841
 842                 for (j = 0; j < i; j++) {
 843                         struct rte_swx_field_params *f_prev = &fields[j];
 844
 845                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 846                 }
 847         }
 848
 849         CHECK(!struct_type_find(p, name), EEXIST);
 850
 851         /* Node allocation. */
 852         st = calloc(1, sizeof(struct struct_type));
 853         CHECK(st, ENOMEM);
 854
 855         st->fields = calloc(n_fields, sizeof(struct field));
 856         if (!st->fields) {
 857                 free(st);
 858                 CHECK(0, ENOMEM);
 859         }
 860
 861         /* Node initialization. */
 862         strcpy(st->name, name);
 863         for (i = 0; i < n_fields; i++) {
 864                 struct field *dst = &st->fields[i];
 865                 struct rte_swx_field_params *src = &fields[i];
 866
 867                 strcpy(dst->name, src->name);
 868                 dst->n_bits = src->n_bits;
 869                 dst->offset = st->n_bits;
 870
 871                 st->n_bits += src->n_bits;
 872         }
 873         st->n_fields = n_fields;
 874
 875         /* Node add to tailq. */
 876         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 877
 878         return 0;
 879 }
 880
 881 static int
 882 struct_build(struct rte_swx_pipeline *p)
 883 {
 884         uint32_t i;
 885
 886         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 887                 struct thread *t = &p->threads[i];
 888
 889                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 890                 CHECK(t->structs, ENOMEM);
 891         }
 892
 893         return 0;
 894 }
 895
 896 static void
 897 struct_build_free(struct rte_swx_pipeline *p)
 898 {
 899         uint32_t i;
 900
 901         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 902                 struct thread *t = &p->threads[i];
 903
 904                 free(t->structs);
 905                 t->structs = NULL;
 906         }
 907 }
 908
 909 static void
 910 struct_free(struct rte_swx_pipeline *p)
 911 {
 912         struct_build_free(p);
 913
 914         /* Struct types. */
 915         for ( ; ; ) {
 916                 struct struct_type *elem;
 917
 918                 elem = TAILQ_FIRST(&p->struct_types);
 919                 if (!elem)
 920                         break;
 921
 922                 TAILQ_REMOVE(&p->struct_types, elem, node);
 923                 free(elem->fields);
 924                 free(elem);
 925         }
 926 }
 927
 928 /*
 929  * Input port.
 930  */
 931 static struct port_in_type *
 932 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 933 {
 934         struct port_in_type *elem;
 935
 936         if (!name)
 937                 return NULL;
 938
 939         TAILQ_FOREACH(elem, &p->port_in_types, node)
 940                 if (strcmp(elem->name, name) == 0)
 941                         return elem;
 942
 943         return NULL;
 944 }
 945
 946 int
 947 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 948                                        const char *name,
 949                                        struct rte_swx_port_in_ops *ops)
 950 {
 951         struct port_in_type *elem;
 952
 953         CHECK(p, EINVAL);
 954         CHECK_NAME(name, EINVAL);
 955         CHECK(ops, EINVAL);
 956         CHECK(ops->create, EINVAL);
 957         CHECK(ops->free, EINVAL);
 958         CHECK(ops->pkt_rx, EINVAL);
 959         CHECK(ops->stats_read, EINVAL);
 960
 961         CHECK(!port_in_type_find(p, name), EEXIST);
 962
 963         /* Node allocation. */
 964         elem = calloc(1, sizeof(struct port_in_type));
 965         CHECK(elem, ENOMEM);
 966
 967         /* Node initialization. */
 968         strcpy(elem->name, name);
 969         memcpy(&elem->ops, ops, sizeof(*ops));
 970
 971         /* Node add to tailq. */
 972         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 973
 974         return 0;
 975 }
 976
 977 static struct port_in *
 978 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 979 {
 980         struct port_in *port;
 981
 982         TAILQ_FOREACH(port, &p->ports_in, node)
 983                 if (port->id == port_id)
 984                         return port;
 985
 986         return NULL;
 987 }
 988
 989 int
 990 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 991                                 uint32_t port_id,
 992                                 const char *port_type_name,
 993                                 void *args)
 994 {
 995         struct port_in_type *type = NULL;
 996         struct port_in *port = NULL;
 997         void *obj = NULL;
 998
 999         CHECK(p, EINVAL);
1000
1001         CHECK(!port_in_find(p, port_id), EINVAL);
1002
1003         CHECK_NAME(port_type_name, EINVAL);
1004         type = port_in_type_find(p, port_type_name);
1005         CHECK(type, EINVAL);
1006
1007         obj = type->ops.create(args);
1008         CHECK(obj, ENODEV);
1009
1010         /* Node allocation. */
1011         port = calloc(1, sizeof(struct port_in));
1012         CHECK(port, ENOMEM);
1013
1014         /* Node initialization. */
1015         port->type = type;
1016         port->obj = obj;
1017         port->id = port_id;
1018
1019         /* Node add to tailq. */
1020         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1021         if (p->n_ports_in < port_id + 1)
1022                 p->n_ports_in = port_id + 1;
1023
1024         return 0;
1025 }
1026
1027 static int
1028 port_in_build(struct rte_swx_pipeline *p)
1029 {
1030         struct port_in *port;
1031         uint32_t i;
1032
1033         CHECK(p->n_ports_in, EINVAL);
1034         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1035
1036         for (i = 0; i < p->n_ports_in; i++)
1037                 CHECK(port_in_find(p, i), EINVAL);
1038
1039         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1040         CHECK(p->in, ENOMEM);
1041
1042         TAILQ_FOREACH(port, &p->ports_in, node) {
1043                 struct port_in_runtime *in = &p->in[port->id];
1044
1045                 in->pkt_rx = port->type->ops.pkt_rx;
1046                 in->obj = port->obj;
1047         }
1048
1049         return 0;
1050 }
1051
1052 static void
1053 port_in_build_free(struct rte_swx_pipeline *p)
1054 {
1055         free(p->in);
1056         p->in = NULL;
1057 }
1058
1059 static void
1060 port_in_free(struct rte_swx_pipeline *p)
1061 {
1062         port_in_build_free(p);
1063
1064         /* Input ports. */
1065         for ( ; ; ) {
1066                 struct port_in *port;
1067
1068                 port = TAILQ_FIRST(&p->ports_in);
1069                 if (!port)
1070                         break;
1071
1072                 TAILQ_REMOVE(&p->ports_in, port, node);
1073                 port->type->ops.free(port->obj);
1074                 free(port);
1075         }
1076
1077         /* Input port types. */
1078         for ( ; ; ) {
1079                 struct port_in_type *elem;
1080
1081                 elem = TAILQ_FIRST(&p->port_in_types);
1082                 if (!elem)
1083                         break;
1084
1085                 TAILQ_REMOVE(&p->port_in_types, elem, node);
1086                 free(elem);
1087         }
1088 }
1089
1090 /*
1091  * Output port.
1092  */
1093 static struct port_out_type *
1094 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1095 {
1096         struct port_out_type *elem;
1097
1098         if (!name)
1099                 return NULL;
1100
1101         TAILQ_FOREACH(elem, &p->port_out_types, node)
1102                 if (!strcmp(elem->name, name))
1103                         return elem;
1104
1105         return NULL;
1106 }
1107
1108 int
1109 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1110                                         const char *name,
1111                                         struct rte_swx_port_out_ops *ops)
1112 {
1113         struct port_out_type *elem;
1114
1115         CHECK(p, EINVAL);
1116         CHECK_NAME(name, EINVAL);
1117         CHECK(ops, EINVAL);
1118         CHECK(ops->create, EINVAL);
1119         CHECK(ops->free, EINVAL);
1120         CHECK(ops->pkt_tx, EINVAL);
1121         CHECK(ops->stats_read, EINVAL);
1122
1123         CHECK(!port_out_type_find(p, name), EEXIST);
1124
1125         /* Node allocation. */
1126         elem = calloc(1, sizeof(struct port_out_type));
1127         CHECK(elem, ENOMEM);
1128
1129         /* Node initialization. */
1130         strcpy(elem->name, name);
1131         memcpy(&elem->ops, ops, sizeof(*ops));
1132
1133         /* Node add to tailq. */
1134         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1135
1136         return 0;
1137 }
1138
1139 static struct port_out *
1140 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1141 {
1142         struct port_out *port;
1143
1144         TAILQ_FOREACH(port, &p->ports_out, node)
1145                 if (port->id == port_id)
1146                         return port;
1147
1148         return NULL;
1149 }
1150
1151 int
1152 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1153                                  uint32_t port_id,
1154                                  const char *port_type_name,
1155                                  void *args)
1156 {
1157         struct port_out_type *type = NULL;
1158         struct port_out *port = NULL;
1159         void *obj = NULL;
1160
1161         CHECK(p, EINVAL);
1162
1163         CHECK(!port_out_find(p, port_id), EINVAL);
1164
1165         CHECK_NAME(port_type_name, EINVAL);
1166         type = port_out_type_find(p, port_type_name);
1167         CHECK(type, EINVAL);
1168
1169         obj = type->ops.create(args);
1170         CHECK(obj, ENODEV);
1171
1172         /* Node allocation. */
1173         port = calloc(1, sizeof(struct port_out));
1174         CHECK(port, ENOMEM);
1175
1176         /* Node initialization. */
1177         port->type = type;
1178         port->obj = obj;
1179         port->id = port_id;
1180
1181         /* Node add to tailq. */
1182         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1183         if (p->n_ports_out < port_id + 1)
1184                 p->n_ports_out = port_id + 1;
1185
1186         return 0;
1187 }
1188
1189 static int
1190 port_out_build(struct rte_swx_pipeline *p)
1191 {
1192         struct port_out *port;
1193         uint32_t i;
1194
1195         CHECK(p->n_ports_out, EINVAL);
1196
1197         for (i = 0; i < p->n_ports_out; i++)
1198                 CHECK(port_out_find(p, i), EINVAL);
1199
1200         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1201         CHECK(p->out, ENOMEM);
1202
1203         TAILQ_FOREACH(port, &p->ports_out, node) {
1204                 struct port_out_runtime *out = &p->out[port->id];
1205
1206                 out->pkt_tx = port->type->ops.pkt_tx;
1207                 out->flush = port->type->ops.flush;
1208                 out->obj = port->obj;
1209         }
1210
1211         return 0;
1212 }
1213
1214 static void
1215 port_out_build_free(struct rte_swx_pipeline *p)
1216 {
1217         free(p->out);
1218         p->out = NULL;
1219 }
1220
1221 static void
1222 port_out_free(struct rte_swx_pipeline *p)
1223 {
1224         port_out_build_free(p);
1225
1226         /* Output ports. */
1227         for ( ; ; ) {
1228                 struct port_out *port;
1229
1230                 port = TAILQ_FIRST(&p->ports_out);
1231                 if (!port)
1232                         break;
1233
1234                 TAILQ_REMOVE(&p->ports_out, port, node);
1235                 port->type->ops.free(port->obj);
1236                 free(port);
1237         }
1238
1239         /* Output port types. */
1240         for ( ; ; ) {
1241                 struct port_out_type *elem;
1242
1243                 elem = TAILQ_FIRST(&p->port_out_types);
1244                 if (!elem)
1245                         break;
1246
1247                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1248                 free(elem);
1249         }
1250 }
1251
1252 /*
1253  * Extern object.
1254  */
1255 static struct extern_type *
1256 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1257 {
1258         struct extern_type *elem;
1259
1260         TAILQ_FOREACH(elem, &p->extern_types, node)
1261                 if (strcmp(elem->name, name) == 0)
1262                         return elem;
1263
1264         return NULL;
1265 }
1266
1267 static struct extern_type_member_func *
1268 extern_type_member_func_find(struct extern_type *type, const char *name)
1269 {
1270         struct extern_type_member_func *elem;
1271
1272         TAILQ_FOREACH(elem, &type->funcs, node)
1273                 if (strcmp(elem->name, name) == 0)
1274                         return elem;
1275
1276         return NULL;
1277 }
1278
1279 static struct extern_obj *
1280 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1281 {
1282         struct extern_obj *elem;
1283
1284         TAILQ_FOREACH(elem, &p->extern_objs, node)
1285                 if (strcmp(elem->name, name) == 0)
1286                         return elem;
1287
1288         return NULL;
1289 }
1290
1291 static struct extern_type_member_func *
1292 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1293                              const char *name,
1294                              struct extern_obj **obj)
1295 {
1296         struct extern_obj *object;
1297         struct extern_type_member_func *func;
1298         char *object_name, *func_name;
1299
1300         if (name[0] != 'e' || name[1] != '.')
1301                 return NULL;
1302
1303         object_name = strdup(&name[2]);
1304         if (!object_name)
1305                 return NULL;
1306
1307         func_name = strchr(object_name, '.');
1308         if (!func_name) {
1309                 free(object_name);
1310                 return NULL;
1311         }
1312
1313         *func_name = 0;
1314         func_name++;
1315
1316         object = extern_obj_find(p, object_name);
1317         if (!object) {
1318                 free(object_name);
1319                 return NULL;
1320         }
1321
1322         func = extern_type_member_func_find(object->type, func_name);
1323         if (!func) {
1324                 free(object_name);
1325                 return NULL;
1326         }
1327
1328         if (obj)
1329                 *obj = object;
1330
1331         free(object_name);
1332         return func;
1333 }
1334
1335 static struct field *
1336 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1337                                const char *name,
1338                                struct extern_obj **object)
1339 {
1340         struct extern_obj *obj;
1341         struct field *f;
1342         char *obj_name, *field_name;
1343
1344         if ((name[0] != 'e') || (name[1] != '.'))
1345                 return NULL;
1346
1347         obj_name = strdup(&name[2]);
1348         if (!obj_name)
1349                 return NULL;
1350
1351         field_name = strchr(obj_name, '.');
1352         if (!field_name) {
1353                 free(obj_name);
1354                 return NULL;
1355         }
1356
1357         *field_name = 0;
1358         field_name++;
1359
1360         obj = extern_obj_find(p, obj_name);
1361         if (!obj) {
1362                 free(obj_name);
1363                 return NULL;
1364         }
1365
1366         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1367         if (!f) {
1368                 free(obj_name);
1369                 return NULL;
1370         }
1371
1372         if (object)
1373                 *object = obj;
1374
1375         free(obj_name);
1376         return f;
1377 }
1378
1379 int
1380 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1381         const char *name,
1382         const char *mailbox_struct_type_name,
1383         rte_swx_extern_type_constructor_t constructor,
1384         rte_swx_extern_type_destructor_t destructor)
1385 {
1386         struct extern_type *elem;
1387         struct struct_type *mailbox_struct_type;
1388
1389         CHECK(p, EINVAL);
1390
1391         CHECK_NAME(name, EINVAL);
1392         CHECK(!extern_type_find(p, name), EEXIST);
1393
1394         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1395         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1396         CHECK(mailbox_struct_type, EINVAL);
1397
1398         CHECK(constructor, EINVAL);
1399         CHECK(destructor, EINVAL);
1400
1401         /* Node allocation. */
1402         elem = calloc(1, sizeof(struct extern_type));
1403         CHECK(elem, ENOMEM);
1404
1405         /* Node initialization. */
1406         strcpy(elem->name, name);
1407         elem->mailbox_struct_type = mailbox_struct_type;
1408         elem->constructor = constructor;
1409         elem->destructor = destructor;
1410         TAILQ_INIT(&elem->funcs);
1411
1412         /* Node add to tailq. */
1413         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1414
1415         return 0;
1416 }
1417
1418 int
1419 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1420         const char *extern_type_name,
1421         const char *name,
1422         rte_swx_extern_type_member_func_t member_func)
1423 {
1424         struct extern_type *type;
1425         struct extern_type_member_func *type_member;
1426
1427         CHECK(p, EINVAL);
1428
1429         CHECK(extern_type_name, EINVAL);
1430         type = extern_type_find(p, extern_type_name);
1431         CHECK(type, EINVAL);
1432         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1433
1434         CHECK(name, EINVAL);
1435         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1436
1437         CHECK(member_func, EINVAL);
1438
1439         /* Node allocation. */
1440         type_member = calloc(1, sizeof(struct extern_type_member_func));
1441         CHECK(type_member, ENOMEM);
1442
1443         /* Node initialization. */
1444         strcpy(type_member->name, name);
1445         type_member->func = member_func;
1446         type_member->id = type->n_funcs;
1447
1448         /* Node add to tailq. */
1449         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1450         type->n_funcs++;
1451
1452         return 0;
1453 }
1454
1455 int
1456 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1457                                       const char *extern_type_name,
1458                                       const char *name,
1459                                       const char *args)
1460 {
1461         struct extern_type *type;
1462         struct extern_obj *obj;
1463         void *obj_handle;
1464
1465         CHECK(p, EINVAL);
1466
1467         CHECK_NAME(extern_type_name, EINVAL);
1468         type = extern_type_find(p, extern_type_name);
1469         CHECK(type, EINVAL);
1470
1471         CHECK_NAME(name, EINVAL);
1472         CHECK(!extern_obj_find(p, name), EEXIST);
1473
1474         /* Node allocation. */
1475         obj = calloc(1, sizeof(struct extern_obj));
1476         CHECK(obj, ENOMEM);
1477
1478         /* Object construction. */
1479         obj_handle = type->constructor(args);
1480         if (!obj_handle) {
1481                 free(obj);
1482                 CHECK(0, ENODEV);
1483         }
1484
1485         /* Node initialization. */
1486         strcpy(obj->name, name);
1487         obj->type = type;
1488         obj->obj = obj_handle;
1489         obj->struct_id = p->n_structs;
1490         obj->id = p->n_extern_objs;
1491
1492         /* Node add to tailq. */
1493         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1494         p->n_extern_objs++;
1495         p->n_structs++;
1496
1497         return 0;
1498 }
1499
1500 static int
1501 extern_obj_build(struct rte_swx_pipeline *p)
1502 {
1503         uint32_t i;
1504
1505         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1506                 struct thread *t = &p->threads[i];
1507                 struct extern_obj *obj;
1508
1509                 t->extern_objs = calloc(p->n_extern_objs,
1510                                         sizeof(struct extern_obj_runtime));
1511                 CHECK(t->extern_objs, ENOMEM);
1512
1513                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1514                         struct extern_obj_runtime *r =
1515                                 &t->extern_objs[obj->id];
1516                         struct extern_type_member_func *func;
1517                         uint32_t mailbox_size =
1518                                 obj->type->mailbox_struct_type->n_bits / 8;
1519
1520                         r->obj = obj->obj;
1521
1522                         r->mailbox = calloc(1, mailbox_size);
1523                         CHECK(r->mailbox, ENOMEM);
1524
1525                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1526                                 r->funcs[func->id] = func->func;
1527
1528                         t->structs[obj->struct_id] = r->mailbox;
1529                 }
1530         }
1531
1532         return 0;
1533 }
1534
1535 static void
1536 extern_obj_build_free(struct rte_swx_pipeline *p)
1537 {
1538         uint32_t i;
1539
1540         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1541                 struct thread *t = &p->threads[i];
1542                 uint32_t j;
1543
1544                 if (!t->extern_objs)
1545                         continue;
1546
1547                 for (j = 0; j < p->n_extern_objs; j++) {
1548                         struct extern_obj_runtime *r = &t->extern_objs[j];
1549
1550                         free(r->mailbox);
1551                 }
1552
1553                 free(t->extern_objs);
1554                 t->extern_objs = NULL;
1555         }
1556 }
1557
1558 static void
1559 extern_obj_free(struct rte_swx_pipeline *p)
1560 {
1561         extern_obj_build_free(p);
1562
1563         /* Extern objects. */
1564         for ( ; ; ) {
1565                 struct extern_obj *elem;
1566
1567                 elem = TAILQ_FIRST(&p->extern_objs);
1568                 if (!elem)
1569                         break;
1570
1571                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1572                 if (elem->obj)
1573                         elem->type->destructor(elem->obj);
1574                 free(elem);
1575         }
1576
1577         /* Extern types. */
1578         for ( ; ; ) {
1579                 struct extern_type *elem;
1580
1581                 elem = TAILQ_FIRST(&p->extern_types);
1582                 if (!elem)
1583                         break;
1584
1585                 TAILQ_REMOVE(&p->extern_types, elem, node);
1586
1587                 for ( ; ; ) {
1588                         struct extern_type_member_func *func;
1589
1590                         func = TAILQ_FIRST(&elem->funcs);
1591                         if (!func)
1592                                 break;
1593
1594                         TAILQ_REMOVE(&elem->funcs, func, node);
1595                         free(func);
1596                 }
1597
1598                 free(elem);
1599         }
1600 }
1601
1602 /*
1603  * Extern function.
1604  */
1605 static struct extern_func *
1606 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1607 {
1608         struct extern_func *elem;
1609
1610         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1611                 if (strcmp(elem->name, name) == 0)
1612                         return elem;
1613
1614         return NULL;
1615 }
1616
1617 static struct extern_func *
1618 extern_func_parse(struct rte_swx_pipeline *p,
1619                   const char *name)
1620 {
1621         if (name[0] != 'f' || name[1] != '.')
1622                 return NULL;
1623
1624         return extern_func_find(p, &name[2]);
1625 }
1626
1627 static struct field *
1628 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1629                                 const char *name,
1630                                 struct extern_func **function)
1631 {
1632         struct extern_func *func;
1633         struct field *f;
1634         char *func_name, *field_name;
1635
1636         if ((name[0] != 'f') || (name[1] != '.'))
1637                 return NULL;
1638
1639         func_name = strdup(&name[2]);
1640         if (!func_name)
1641                 return NULL;
1642
1643         field_name = strchr(func_name, '.');
1644         if (!field_name) {
1645                 free(func_name);
1646                 return NULL;
1647         }
1648
1649         *field_name = 0;
1650         field_name++;
1651
1652         func = extern_func_find(p, func_name);
1653         if (!func) {
1654                 free(func_name);
1655                 return NULL;
1656         }
1657
1658         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1659         if (!f) {
1660                 free(func_name);
1661                 return NULL;
1662         }
1663
1664         if (function)
1665                 *function = func;
1666
1667         free(func_name);
1668         return f;
1669 }
1670
1671 int
1672 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1673                                       const char *name,
1674                                       const char *mailbox_struct_type_name,
1675                                       rte_swx_extern_func_t func)
1676 {
1677         struct extern_func *f;
1678         struct struct_type *mailbox_struct_type;
1679
1680         CHECK(p, EINVAL);
1681
1682         CHECK_NAME(name, EINVAL);
1683         CHECK(!extern_func_find(p, name), EEXIST);
1684
1685         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1686         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1687         CHECK(mailbox_struct_type, EINVAL);
1688
1689         CHECK(func, EINVAL);
1690
1691         /* Node allocation. */
1692         f = calloc(1, sizeof(struct extern_func));
1693         CHECK(func, ENOMEM);
1694
1695         /* Node initialization. */
1696         strcpy(f->name, name);
1697         f->mailbox_struct_type = mailbox_struct_type;
1698         f->func = func;
1699         f->struct_id = p->n_structs;
1700         f->id = p->n_extern_funcs;
1701
1702         /* Node add to tailq. */
1703         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1704         p->n_extern_funcs++;
1705         p->n_structs++;
1706
1707         return 0;
1708 }
1709
1710 static int
1711 extern_func_build(struct rte_swx_pipeline *p)
1712 {
1713         uint32_t i;
1714
1715         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1716                 struct thread *t = &p->threads[i];
1717                 struct extern_func *func;
1718
1719                 /* Memory allocation. */
1720                 t->extern_funcs = calloc(p->n_extern_funcs,
1721                                          sizeof(struct extern_func_runtime));
1722                 CHECK(t->extern_funcs, ENOMEM);
1723
1724                 /* Extern function. */
1725                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1726                         struct extern_func_runtime *r =
1727                                 &t->extern_funcs[func->id];
1728                         uint32_t mailbox_size =
1729                                 func->mailbox_struct_type->n_bits / 8;
1730
1731                         r->func = func->func;
1732
1733                         r->mailbox = calloc(1, mailbox_size);
1734                         CHECK(r->mailbox, ENOMEM);
1735
1736                         t->structs[func->struct_id] = r->mailbox;
1737                 }
1738         }
1739
1740         return 0;
1741 }
1742
1743 static void
1744 extern_func_build_free(struct rte_swx_pipeline *p)
1745 {
1746         uint32_t i;
1747
1748         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1749                 struct thread *t = &p->threads[i];
1750                 uint32_t j;
1751
1752                 if (!t->extern_funcs)
1753                         continue;
1754
1755                 for (j = 0; j < p->n_extern_funcs; j++) {
1756                         struct extern_func_runtime *r = &t->extern_funcs[j];
1757
1758                         free(r->mailbox);
1759                 }
1760
1761                 free(t->extern_funcs);
1762                 t->extern_funcs = NULL;
1763         }
1764 }
1765
1766 static void
1767 extern_func_free(struct rte_swx_pipeline *p)
1768 {
1769         extern_func_build_free(p);
1770
1771         for ( ; ; ) {
1772                 struct extern_func *elem;
1773
1774                 elem = TAILQ_FIRST(&p->extern_funcs);
1775                 if (!elem)
1776                         break;
1777
1778                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1779                 free(elem);
1780         }
1781 }
1782
1783 /*
1784  * Header.
1785  */
1786 static struct header *
1787 header_find(struct rte_swx_pipeline *p, const char *name)
1788 {
1789         struct header *elem;
1790
1791         TAILQ_FOREACH(elem, &p->headers, node)
1792                 if (strcmp(elem->name, name) == 0)
1793                         return elem;
1794
1795         return NULL;
1796 }
1797
1798 static struct header *
1799 header_parse(struct rte_swx_pipeline *p,
1800              const char *name)
1801 {
1802         if (name[0] != 'h' || name[1] != '.')
1803                 return NULL;
1804
1805         return header_find(p, &name[2]);
1806 }
1807
1808 static struct field *
1809 header_field_parse(struct rte_swx_pipeline *p,
1810                    const char *name,
1811                    struct header **header)
1812 {
1813         struct header *h;
1814         struct field *f;
1815         char *header_name, *field_name;
1816
1817         if ((name[0] != 'h') || (name[1] != '.'))
1818                 return NULL;
1819
1820         header_name = strdup(&name[2]);
1821         if (!header_name)
1822                 return NULL;
1823
1824         field_name = strchr(header_name, '.');
1825         if (!field_name) {
1826                 free(header_name);
1827                 return NULL;
1828         }
1829
1830         *field_name = 0;
1831         field_name++;
1832
1833         h = header_find(p, header_name);
1834         if (!h) {
1835                 free(header_name);
1836                 return NULL;
1837         }
1838
1839         f = struct_type_field_find(h->st, field_name);
1840         if (!f) {
1841                 free(header_name);
1842                 return NULL;
1843         }
1844
1845         if (header)
1846                 *header = h;
1847
1848         free(header_name);
1849         return f;
1850 }
1851
1852 int
1853 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1854                                         const char *name,
1855                                         const char *struct_type_name)
1856 {
1857         struct struct_type *st;
1858         struct header *h;
1859         size_t n_headers_max;
1860
1861         CHECK(p, EINVAL);
1862         CHECK_NAME(name, EINVAL);
1863         CHECK_NAME(struct_type_name, EINVAL);
1864
1865         CHECK(!header_find(p, name), EEXIST);
1866
1867         st = struct_type_find(p, struct_type_name);
1868         CHECK(st, EINVAL);
1869
1870         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1871         CHECK(p->n_headers < n_headers_max, ENOSPC);
1872
1873         /* Node allocation. */
1874         h = calloc(1, sizeof(struct header));
1875         CHECK(h, ENOMEM);
1876
1877         /* Node initialization. */
1878         strcpy(h->name, name);
1879         h->st = st;
1880         h->struct_id = p->n_structs;
1881         h->id = p->n_headers;
1882
1883         /* Node add to tailq. */
1884         TAILQ_INSERT_TAIL(&p->headers, h, node);
1885         p->n_headers++;
1886         p->n_structs++;
1887
1888         return 0;
1889 }
1890
1891 static int
1892 header_build(struct rte_swx_pipeline *p)
1893 {
1894         struct header *h;
1895         uint32_t n_bytes = 0, i;
1896
1897         TAILQ_FOREACH(h, &p->headers, node) {
1898                 n_bytes += h->st->n_bits / 8;
1899         }
1900
1901         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1902                 struct thread *t = &p->threads[i];
1903                 uint32_t offset = 0;
1904
1905                 t->headers = calloc(p->n_headers,
1906                                     sizeof(struct header_runtime));
1907                 CHECK(t->headers, ENOMEM);
1908
1909                 t->headers_out = calloc(p->n_headers,
1910                                         sizeof(struct header_out_runtime));
1911                 CHECK(t->headers_out, ENOMEM);
1912
1913                 t->header_storage = calloc(1, n_bytes);
1914                 CHECK(t->header_storage, ENOMEM);
1915
1916                 t->header_out_storage = calloc(1, n_bytes);
1917                 CHECK(t->header_out_storage, ENOMEM);
1918
1919                 TAILQ_FOREACH(h, &p->headers, node) {
1920                         uint8_t *header_storage;
1921
1922                         header_storage = &t->header_storage[offset];
1923                         offset += h->st->n_bits / 8;
1924
1925                         t->headers[h->id].ptr0 = header_storage;
1926                         t->structs[h->struct_id] = header_storage;
1927                 }
1928         }
1929
1930         return 0;
1931 }
1932
1933 static void
1934 header_build_free(struct rte_swx_pipeline *p)
1935 {
1936         uint32_t i;
1937
1938         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1939                 struct thread *t = &p->threads[i];
1940
1941                 free(t->headers_out);
1942                 t->headers_out = NULL;
1943
1944                 free(t->headers);
1945                 t->headers = NULL;
1946
1947                 free(t->header_out_storage);
1948                 t->header_out_storage = NULL;
1949
1950                 free(t->header_storage);
1951                 t->header_storage = NULL;
1952         }
1953 }
1954
1955 static void
1956 header_free(struct rte_swx_pipeline *p)
1957 {
1958         header_build_free(p);
1959
1960         for ( ; ; ) {
1961                 struct header *elem;
1962
1963                 elem = TAILQ_FIRST(&p->headers);
1964                 if (!elem)
1965                         break;
1966
1967                 TAILQ_REMOVE(&p->headers, elem, node);
1968                 free(elem);
1969         }
1970 }
1971
1972 /*
1973  * Meta-data.
1974  */
1975 static struct field *
1976 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1977 {
1978         if (!p->metadata_st)
1979                 return NULL;
1980
1981         if (name[0] != 'm' || name[1] != '.')
1982                 return NULL;
1983
1984         return struct_type_field_find(p->metadata_st, &name[2]);
1985 }
1986
1987 int
1988 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1989                                           const char *struct_type_name)
1990 {
1991         struct struct_type *st = NULL;
1992
1993         CHECK(p, EINVAL);
1994
1995         CHECK_NAME(struct_type_name, EINVAL);
1996         st  = struct_type_find(p, struct_type_name);
1997         CHECK(st, EINVAL);
1998         CHECK(!p->metadata_st, EINVAL);
1999
2000         p->metadata_st = st;
2001         p->metadata_struct_id = p->n_structs;
2002
2003         p->n_structs++;
2004
2005         return 0;
2006 }
2007
2008 static int
2009 metadata_build(struct rte_swx_pipeline *p)
2010 {
2011         uint32_t n_bytes = p->metadata_st->n_bits / 8;
2012         uint32_t i;
2013
2014         /* Thread-level initialization. */
2015         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2016                 struct thread *t = &p->threads[i];
2017                 uint8_t *metadata;
2018
2019                 metadata = calloc(1, n_bytes);
2020                 CHECK(metadata, ENOMEM);
2021
2022                 t->metadata = metadata;
2023                 t->structs[p->metadata_struct_id] = metadata;
2024         }
2025
2026         return 0;
2027 }
2028
2029 static void
2030 metadata_build_free(struct rte_swx_pipeline *p)
2031 {
2032         uint32_t i;
2033
2034         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2035                 struct thread *t = &p->threads[i];
2036
2037                 free(t->metadata);
2038                 t->metadata = NULL;
2039         }
2040 }
2041
2042 static void
2043 metadata_free(struct rte_swx_pipeline *p)
2044 {
2045         metadata_build_free(p);
2046 }
2047
2048 /*
2049  * Instruction.
2050  */
2051 static struct field *
2052 action_field_parse(struct action *action, const char *name);
2053
2054 static struct field *
2055 struct_field_parse(struct rte_swx_pipeline *p,
2056                    struct action *action,
2057                    const char *name,
2058                    uint32_t *struct_id)
2059 {
2060         struct field *f;
2061
2062         switch (name[0]) {
2063         case 'h':
2064         {
2065                 struct header *header;
2066
2067                 f = header_field_parse(p, name, &header);
2068                 if (!f)
2069                         return NULL;
2070
2071                 *struct_id = header->struct_id;
2072                 return f;
2073         }
2074
2075         case 'm':
2076         {
2077                 f = metadata_field_parse(p, name);
2078                 if (!f)
2079                         return NULL;
2080
2081                 *struct_id = p->metadata_struct_id;
2082                 return f;
2083         }
2084
2085         case 't':
2086         {
2087                 if (!action)
2088                         return NULL;
2089
2090                 f = action_field_parse(action, name);
2091                 if (!f)
2092                         return NULL;
2093
2094                 *struct_id = 0;
2095                 return f;
2096         }
2097
2098         case 'e':
2099         {
2100                 struct extern_obj *obj;
2101
2102                 f = extern_obj_mailbox_field_parse(p, name, &obj);
2103                 if (!f)
2104                         return NULL;
2105
2106                 *struct_id = obj->struct_id;
2107                 return f;
2108         }
2109
2110         case 'f':
2111         {
2112                 struct extern_func *func;
2113
2114                 f = extern_func_mailbox_field_parse(p, name, &func);
2115                 if (!f)
2116                         return NULL;
2117
2118                 *struct_id = func->struct_id;
2119                 return f;
2120         }
2121
2122         default:
2123                 return NULL;
2124         }
2125 }
2126
2127 static inline void
2128 pipeline_port_inc(struct rte_swx_pipeline *p)
2129 {
2130         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2131 }
2132
2133 static inline void
2134 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2135 {
2136         t->ip = p->instructions;
2137 }
2138
2139 static inline void
2140 thread_ip_action_call(struct rte_swx_pipeline *p,
2141                       struct thread *t,
2142                       uint32_t action_id)
2143 {
2144         t->ret = t->ip + 1;
2145         t->ip = p->action_instructions[action_id];
2146 }
2147
2148 static inline void
2149 thread_ip_inc(struct rte_swx_pipeline *p);
2150
2151 static inline void
2152 thread_ip_inc(struct rte_swx_pipeline *p)
2153 {
2154         struct thread *t = &p->threads[p->thread_id];
2155
2156         t->ip++;
2157 }
2158
2159 static inline void
2160 thread_ip_inc_cond(struct thread *t, int cond)
2161 {
2162         t->ip += cond;
2163 }
2164
2165 static inline void
2166 thread_yield(struct rte_swx_pipeline *p)
2167 {
2168         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2169 }
2170
2171 static inline void
2172 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2173 {
2174         p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2175 }
2176
2177 /*
2178  * rx.
2179  */
2180 static int
2181 instr_rx_translate(struct rte_swx_pipeline *p,
2182                    struct action *action,
2183                    char **tokens,
2184                    int n_tokens,
2185                    struct instruction *instr,
2186                    struct instruction_data *data __rte_unused)
2187 {
2188         struct field *f;
2189
2190         CHECK(!action, EINVAL);
2191         CHECK(n_tokens == 2, EINVAL);
2192
2193         f = metadata_field_parse(p, tokens[1]);
2194         CHECK(f, EINVAL);
2195
2196         instr->type = INSTR_RX;
2197         instr->io.io.offset = f->offset / 8;
2198         instr->io.io.n_bits = f->n_bits;
2199         return 0;
2200 }
2201
2202 static inline void
2203 instr_rx_exec(struct rte_swx_pipeline *p);
2204
2205 static inline void
2206 instr_rx_exec(struct rte_swx_pipeline *p)
2207 {
2208         struct thread *t = &p->threads[p->thread_id];
2209         struct instruction *ip = t->ip;
2210         struct port_in_runtime *port = &p->in[p->port_id];
2211         struct rte_swx_pkt *pkt = &t->pkt;
2212         int pkt_received;
2213
2214         /* Packet. */
2215         pkt_received = port->pkt_rx(port->obj, pkt);
2216         t->ptr = &pkt->pkt[pkt->offset];
2217         rte_prefetch0(t->ptr);
2218
2219         TRACE("[Thread %2u] rx %s from port %u\n",
2220               p->thread_id,
2221               pkt_received ? "1 pkt" : "0 pkts",
2222               p->port_id);
2223
2224         /* Headers. */
2225         t->valid_headers = 0;
2226         t->n_headers_out = 0;
2227
2228         /* Meta-data. */
2229         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2230
2231         /* Tables. */
2232         t->table_state = p->table_state;
2233
2234         /* Thread. */
2235         pipeline_port_inc(p);
2236         thread_ip_inc_cond(t, pkt_received);
2237         thread_yield(p);
2238 }
2239
2240 /*
2241  * tx.
2242  */
2243 static int
2244 instr_tx_translate(struct rte_swx_pipeline *p,
2245                    struct action *action __rte_unused,
2246                    char **tokens,
2247                    int n_tokens,
2248                    struct instruction *instr,
2249                    struct instruction_data *data __rte_unused)
2250 {
2251         struct field *f;
2252
2253         CHECK(n_tokens == 2, EINVAL);
2254
2255         f = metadata_field_parse(p, tokens[1]);
2256         CHECK(f, EINVAL);
2257
2258         instr->type = INSTR_TX;
2259         instr->io.io.offset = f->offset / 8;
2260         instr->io.io.n_bits = f->n_bits;
2261         return 0;
2262 }
2263
2264 static inline void
2265 emit_handler(struct thread *t)
2266 {
2267         struct header_out_runtime *h0 = &t->headers_out[0];
2268         struct header_out_runtime *h1 = &t->headers_out[1];
2269         uint32_t offset = 0, i;
2270
2271         /* No header change or header decapsulation. */
2272         if ((t->n_headers_out == 1) &&
2273             (h0->ptr + h0->n_bytes == t->ptr)) {
2274                 TRACE("Emit handler: no header change or header decap.\n");
2275
2276                 t->pkt.offset -= h0->n_bytes;
2277                 t->pkt.length += h0->n_bytes;
2278
2279                 return;
2280         }
2281
2282         /* Header encapsulation (optionally, with prior header decasulation). */
2283         if ((t->n_headers_out == 2) &&
2284             (h1->ptr + h1->n_bytes == t->ptr) &&
2285             (h0->ptr == h0->ptr0)) {
2286                 uint32_t offset;
2287
2288                 TRACE("Emit handler: header encapsulation.\n");
2289
2290                 offset = h0->n_bytes + h1->n_bytes;
2291                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2292                 t->pkt.offset -= offset;
2293                 t->pkt.length += offset;
2294
2295                 return;
2296         }
2297
2298         /* Header insertion. */
2299         /* TBD */
2300
2301         /* Header extraction. */
2302         /* TBD */
2303
2304         /* For any other case. */
2305         TRACE("Emit handler: complex case.\n");
2306
2307         for (i = 0; i < t->n_headers_out; i++) {
2308                 struct header_out_runtime *h = &t->headers_out[i];
2309
2310                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2311                 offset += h->n_bytes;
2312         }
2313
2314         if (offset) {
2315                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2316                 t->pkt.offset -= offset;
2317                 t->pkt.length += offset;
2318         }
2319 }
2320
2321 static inline void
2322 instr_tx_exec(struct rte_swx_pipeline *p);
2323
2324 static inline void
2325 instr_tx_exec(struct rte_swx_pipeline *p)
2326 {
2327         struct thread *t = &p->threads[p->thread_id];
2328         struct instruction *ip = t->ip;
2329         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2330         struct port_out_runtime *port = &p->out[port_id];
2331         struct rte_swx_pkt *pkt = &t->pkt;
2332
2333         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2334               p->thread_id,
2335               (uint32_t)port_id);
2336
2337         /* Headers. */
2338         emit_handler(t);
2339
2340         /* Packet. */
2341         port->pkt_tx(port->obj, pkt);
2342
2343         /* Thread. */
2344         thread_ip_reset(p, t);
2345         instr_rx_exec(p);
2346 }
2347
2348 /*
2349  * extract.
2350  */
2351 static int
2352 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2353                             struct action *action,
2354                             char **tokens,
2355                             int n_tokens,
2356                             struct instruction *instr,
2357                             struct instruction_data *data __rte_unused)
2358 {
2359         struct header *h;
2360
2361         CHECK(!action, EINVAL);
2362         CHECK(n_tokens == 2, EINVAL);
2363
2364         h = header_parse(p, tokens[1]);
2365         CHECK(h, EINVAL);
2366
2367         instr->type = INSTR_HDR_EXTRACT;
2368         instr->io.hdr.header_id[0] = h->id;
2369         instr->io.hdr.struct_id[0] = h->struct_id;
2370         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2371         return 0;
2372 }
2373
2374 static inline void
2375 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2376
2377 static inline void
2378 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2379 {
2380         struct thread *t = &p->threads[p->thread_id];
2381         struct instruction *ip = t->ip;
2382         uint64_t valid_headers = t->valid_headers;
2383         uint8_t *ptr = t->ptr;
2384         uint32_t offset = t->pkt.offset;
2385         uint32_t length = t->pkt.length;
2386         uint32_t i;
2387
2388         for (i = 0; i < n_extract; i++) {
2389                 uint32_t header_id = ip->io.hdr.header_id[i];
2390                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2391                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2392
2393                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2394                       p->thread_id,
2395                       header_id,
2396                       n_bytes);
2397
2398                 /* Headers. */
2399                 t->structs[struct_id] = ptr;
2400                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2401
2402                 /* Packet. */
2403                 offset += n_bytes;
2404                 length -= n_bytes;
2405                 ptr += n_bytes;
2406         }
2407
2408         /* Headers. */
2409         t->valid_headers = valid_headers;
2410
2411         /* Packet. */
2412         t->pkt.offset = offset;
2413         t->pkt.length = length;
2414         t->ptr = ptr;
2415 }
2416
2417 static inline void
2418 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2419 {
2420         __instr_hdr_extract_exec(p, 1);
2421
2422         /* Thread. */
2423         thread_ip_inc(p);
2424 }
2425
2426 static inline void
2427 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2428 {
2429         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2430               p->thread_id);
2431
2432         __instr_hdr_extract_exec(p, 2);
2433
2434         /* Thread. */
2435         thread_ip_inc(p);
2436 }
2437
2438 static inline void
2439 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2440 {
2441         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2442               p->thread_id);
2443
2444         __instr_hdr_extract_exec(p, 3);
2445
2446         /* Thread. */
2447         thread_ip_inc(p);
2448 }
2449
2450 static inline void
2451 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2452 {
2453         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2454               p->thread_id);
2455
2456         __instr_hdr_extract_exec(p, 4);
2457
2458         /* Thread. */
2459         thread_ip_inc(p);
2460 }
2461
2462 static inline void
2463 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2464 {
2465         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2466               p->thread_id);
2467
2468         __instr_hdr_extract_exec(p, 5);
2469
2470         /* Thread. */
2471         thread_ip_inc(p);
2472 }
2473
2474 static inline void
2475 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2476 {
2477         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2478               p->thread_id);
2479
2480         __instr_hdr_extract_exec(p, 6);
2481
2482         /* Thread. */
2483         thread_ip_inc(p);
2484 }
2485
2486 static inline void
2487 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2488 {
2489         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2490               p->thread_id);
2491
2492         __instr_hdr_extract_exec(p, 7);
2493
2494         /* Thread. */
2495         thread_ip_inc(p);
2496 }
2497
2498 static inline void
2499 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2500 {
2501         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2502               p->thread_id);
2503
2504         __instr_hdr_extract_exec(p, 8);
2505
2506         /* Thread. */
2507         thread_ip_inc(p);
2508 }
2509
2510 /*
2511  * emit.
2512  */
2513 static int
2514 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2515                          struct action *action __rte_unused,
2516                          char **tokens,
2517                          int n_tokens,
2518                          struct instruction *instr,
2519                          struct instruction_data *data __rte_unused)
2520 {
2521         struct header *h;
2522
2523         CHECK(n_tokens == 2, EINVAL);
2524
2525         h = header_parse(p, tokens[1]);
2526         CHECK(h, EINVAL);
2527
2528         instr->type = INSTR_HDR_EMIT;
2529         instr->io.hdr.header_id[0] = h->id;
2530         instr->io.hdr.struct_id[0] = h->struct_id;
2531         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2532         return 0;
2533 }
2534
2535 static inline void
2536 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
2537
2538 static inline void
2539 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
2540 {
2541         struct thread *t = &p->threads[p->thread_id];
2542         struct instruction *ip = t->ip;
2543         uint32_t n_headers_out = t->n_headers_out;
2544         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
2545         uint8_t *ho_ptr = NULL;
2546         uint32_t ho_nbytes = 0, i;
2547
2548         for (i = 0; i < n_emit; i++) {
2549                 uint32_t header_id = ip->io.hdr.header_id[i];
2550                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2551                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2552
2553                 struct header_runtime *hi = &t->headers[header_id];
2554                 uint8_t *hi_ptr = t->structs[struct_id];
2555
2556                 TRACE("[Thread %2u]: emit header %u\n",
2557                       p->thread_id,
2558                       header_id);
2559
2560                 /* Headers. */
2561                 if (!i) {
2562                         if (!t->n_headers_out) {
2563                                 ho = &t->headers_out[0];
2564
2565                                 ho->ptr0 = hi->ptr0;
2566                                 ho->ptr = hi_ptr;
2567
2568                                 ho_ptr = hi_ptr;
2569                                 ho_nbytes = n_bytes;
2570
2571                                 n_headers_out = 1;
2572
2573                                 continue;
2574                         } else {
2575                                 ho_ptr = ho->ptr;
2576                                 ho_nbytes = ho->n_bytes;
2577                         }
2578                 }
2579
2580                 if (ho_ptr + ho_nbytes == hi_ptr) {
2581                         ho_nbytes += n_bytes;
2582                 } else {
2583                         ho->n_bytes = ho_nbytes;
2584
2585                         ho++;
2586                         ho->ptr0 = hi->ptr0;
2587                         ho->ptr = hi_ptr;
2588
2589                         ho_ptr = hi_ptr;
2590                         ho_nbytes = n_bytes;
2591
2592                         n_headers_out++;
2593                 }
2594         }
2595
2596         ho->n_bytes = ho_nbytes;
2597         t->n_headers_out = n_headers_out;
2598 }
2599
2600 static inline void
2601 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2602 {
2603         __instr_hdr_emit_exec(p, 1);
2604
2605         /* Thread. */
2606         thread_ip_inc(p);
2607 }
2608
2609 static inline void
2610 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2611 {
2612         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2613               p->thread_id);
2614
2615         __instr_hdr_emit_exec(p, 1);
2616         instr_tx_exec(p);
2617 }
2618
2619 static inline void
2620 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2621 {
2622         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2623               p->thread_id);
2624
2625         __instr_hdr_emit_exec(p, 2);
2626         instr_tx_exec(p);
2627 }
2628
2629 static inline void
2630 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2631 {
2632         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2633               p->thread_id);
2634
2635         __instr_hdr_emit_exec(p, 3);
2636         instr_tx_exec(p);
2637 }
2638
2639 static inline void
2640 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2641 {
2642         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2643               p->thread_id);
2644
2645         __instr_hdr_emit_exec(p, 4);
2646         instr_tx_exec(p);
2647 }
2648
2649 static inline void
2650 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2651 {
2652         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2653               p->thread_id);
2654
2655         __instr_hdr_emit_exec(p, 5);
2656         instr_tx_exec(p);
2657 }
2658
2659 static inline void
2660 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2661 {
2662         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2663               p->thread_id);
2664
2665         __instr_hdr_emit_exec(p, 6);
2666         instr_tx_exec(p);
2667 }
2668
2669 static inline void
2670 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2671 {
2672         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2673               p->thread_id);
2674
2675         __instr_hdr_emit_exec(p, 7);
2676         instr_tx_exec(p);
2677 }
2678
2679 static inline void
2680 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2681 {
2682         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2683               p->thread_id);
2684
2685         __instr_hdr_emit_exec(p, 8);
2686         instr_tx_exec(p);
2687 }
2688
2689 /*
2690  * validate.
2691  */
2692 static int
2693 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2694                              struct action *action __rte_unused,
2695                              char **tokens,
2696                              int n_tokens,
2697                              struct instruction *instr,
2698                              struct instruction_data *data __rte_unused)
2699 {
2700         struct header *h;
2701
2702         CHECK(n_tokens == 2, EINVAL);
2703
2704         h = header_parse(p, tokens[1]);
2705         CHECK(h, EINVAL);
2706
2707         instr->type = INSTR_HDR_VALIDATE;
2708         instr->valid.header_id = h->id;
2709         return 0;
2710 }
2711
2712 static inline void
2713 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2714 {
2715         struct thread *t = &p->threads[p->thread_id];
2716         struct instruction *ip = t->ip;
2717         uint32_t header_id = ip->valid.header_id;
2718
2719         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2720
2721         /* Headers. */
2722         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2723
2724         /* Thread. */
2725         thread_ip_inc(p);
2726 }
2727
2728 /*
2729  * invalidate.
2730  */
2731 static int
2732 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2733                                struct action *action __rte_unused,
2734                                char **tokens,
2735                                int n_tokens,
2736                                struct instruction *instr,
2737                                struct instruction_data *data __rte_unused)
2738 {
2739         struct header *h;
2740
2741         CHECK(n_tokens == 2, EINVAL);
2742
2743         h = header_parse(p, tokens[1]);
2744         CHECK(h, EINVAL);
2745
2746         instr->type = INSTR_HDR_INVALIDATE;
2747         instr->valid.header_id = h->id;
2748         return 0;
2749 }
2750
2751 static inline void
2752 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2753 {
2754         struct thread *t = &p->threads[p->thread_id];
2755         struct instruction *ip = t->ip;
2756         uint32_t header_id = ip->valid.header_id;
2757
2758         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2759
2760         /* Headers. */
2761         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2762
2763         /* Thread. */
2764         thread_ip_inc(p);
2765 }
2766
2767 /*
2768  * table.
2769  */
2770 static struct table *
2771 table_find(struct rte_swx_pipeline *p, const char *name);
2772
2773 static int
2774 instr_table_translate(struct rte_swx_pipeline *p,
2775                       struct action *action,
2776                       char **tokens,
2777                       int n_tokens,
2778                       struct instruction *instr,
2779                       struct instruction_data *data __rte_unused)
2780 {
2781         struct table *t;
2782
2783         CHECK(!action, EINVAL);
2784         CHECK(n_tokens == 2, EINVAL);
2785
2786         t = table_find(p, tokens[1]);
2787         CHECK(t, EINVAL);
2788
2789         instr->type = INSTR_TABLE;
2790         instr->table.table_id = t->id;
2791         return 0;
2792 }
2793
2794 static inline void
2795 instr_table_exec(struct rte_swx_pipeline *p)
2796 {
2797         struct thread *t = &p->threads[p->thread_id];
2798         struct instruction *ip = t->ip;
2799         uint32_t table_id = ip->table.table_id;
2800         struct rte_swx_table_state *ts = &t->table_state[table_id];
2801         struct table_runtime *table = &t->tables[table_id];
2802         uint64_t action_id;
2803         uint8_t *action_data;
2804         int done, hit;
2805
2806         /* Table. */
2807         done = table->func(ts->obj,
2808                            table->mailbox,
2809                            table->key,
2810                            &action_id,
2811                            &action_data,
2812                            &hit);
2813         if (!done) {
2814                 /* Thread. */
2815                 TRACE("[Thread %2u] table %u (not finalized)\n",
2816                       p->thread_id,
2817                       table_id);
2818
2819                 thread_yield(p);
2820                 return;
2821         }
2822
2823         action_id = hit ? action_id : ts->default_action_id;
2824         action_data = hit ? action_data : ts->default_action_data;
2825
2826         TRACE("[Thread %2u] table %u (%s, action %u)\n",
2827               p->thread_id,
2828               table_id,
2829               hit ? "hit" : "miss",
2830               (uint32_t)action_id);
2831
2832         t->action_id = action_id;
2833         t->structs[0] = action_data;
2834         t->hit = hit;
2835
2836         /* Thread. */
2837         thread_ip_action_call(p, t, action_id);
2838 }
2839
2840 /*
2841  * extern.
2842  */
2843 static int
2844 instr_extern_translate(struct rte_swx_pipeline *p,
2845                        struct action *action __rte_unused,
2846                        char **tokens,
2847                        int n_tokens,
2848                        struct instruction *instr,
2849                        struct instruction_data *data __rte_unused)
2850 {
2851         char *token = tokens[1];
2852
2853         CHECK(n_tokens == 2, EINVAL);
2854
2855         if (token[0] == 'e') {
2856                 struct extern_obj *obj;
2857                 struct extern_type_member_func *func;
2858
2859                 func = extern_obj_member_func_parse(p, token, &obj);
2860                 CHECK(func, EINVAL);
2861
2862                 instr->type = INSTR_EXTERN_OBJ;
2863                 instr->ext_obj.ext_obj_id = obj->id;
2864                 instr->ext_obj.func_id = func->id;
2865
2866                 return 0;
2867         }
2868
2869         if (token[0] == 'f') {
2870                 struct extern_func *func;
2871
2872                 func = extern_func_parse(p, token);
2873                 CHECK(func, EINVAL);
2874
2875                 instr->type = INSTR_EXTERN_FUNC;
2876                 instr->ext_func.ext_func_id = func->id;
2877
2878                 return 0;
2879         }
2880
2881         CHECK(0, EINVAL);
2882 }
2883
2884 static inline void
2885 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2886 {
2887         struct thread *t = &p->threads[p->thread_id];
2888         struct instruction *ip = t->ip;
2889         uint32_t obj_id = ip->ext_obj.ext_obj_id;
2890         uint32_t func_id = ip->ext_obj.func_id;
2891         struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2892         rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2893
2894         TRACE("[Thread %2u] extern obj %u member func %u\n",
2895               p->thread_id,
2896               obj_id,
2897               func_id);
2898
2899         /* Extern object member function execute. */
2900         uint32_t done = func(obj->obj, obj->mailbox);
2901
2902         /* Thread. */
2903         thread_ip_inc_cond(t, done);
2904         thread_yield_cond(p, done ^ 1);
2905 }
2906
2907 static inline void
2908 instr_extern_func_exec(struct rte_swx_pipeline *p)
2909 {
2910         struct thread *t = &p->threads[p->thread_id];
2911         struct instruction *ip = t->ip;
2912         uint32_t ext_func_id = ip->ext_func.ext_func_id;
2913         struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2914         rte_swx_extern_func_t func = ext_func->func;
2915
2916         TRACE("[Thread %2u] extern func %u\n",
2917               p->thread_id,
2918               ext_func_id);
2919
2920         /* Extern function execute. */
2921         uint32_t done = func(ext_func->mailbox);
2922
2923         /* Thread. */
2924         thread_ip_inc_cond(t, done);
2925         thread_yield_cond(p, done ^ 1);
2926 }
2927
2928 /*
2929  * mov.
2930  */
2931 static int
2932 instr_mov_translate(struct rte_swx_pipeline *p,
2933                     struct action *action,
2934                     char **tokens,
2935                     int n_tokens,
2936                     struct instruction *instr,
2937                     struct instruction_data *data __rte_unused)
2938 {
2939         char *dst = tokens[1], *src = tokens[2];
2940         struct field *fdst, *fsrc;
2941         uint32_t dst_struct_id, src_struct_id, src_val;
2942
2943         CHECK(n_tokens == 3, EINVAL);
2944
2945         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2946         CHECK(fdst, EINVAL);
2947
2948         /* MOV or MOV_S. */
2949         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2950         if (fsrc) {
2951                 instr->type = INSTR_MOV;
2952                 if ((dst[0] == 'h' && src[0] != 'h') ||
2953                     (dst[0] != 'h' && src[0] == 'h'))
2954                         instr->type = INSTR_MOV_S;
2955
2956                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2957                 instr->mov.dst.n_bits = fdst->n_bits;
2958                 instr->mov.dst.offset = fdst->offset / 8;
2959                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2960                 instr->mov.src.n_bits = fsrc->n_bits;
2961                 instr->mov.src.offset = fsrc->offset / 8;
2962                 return 0;
2963         }
2964
2965         /* MOV_I. */
2966         src_val = strtoul(src, &src, 0);
2967         CHECK(!src[0], EINVAL);
2968
2969         if (dst[0] == 'h')
2970                 src_val = htonl(src_val);
2971
2972         instr->type = INSTR_MOV_I;
2973         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2974         instr->mov.dst.n_bits = fdst->n_bits;
2975         instr->mov.dst.offset = fdst->offset / 8;
2976         instr->mov.src_val = (uint32_t)src_val;
2977         return 0;
2978 }
2979
2980 static inline void
2981 instr_mov_exec(struct rte_swx_pipeline *p)
2982 {
2983         struct thread *t = &p->threads[p->thread_id];
2984         struct instruction *ip = t->ip;
2985
2986         TRACE("[Thread %2u] mov\n",
2987               p->thread_id);
2988
2989         MOV(t, ip);
2990
2991         /* Thread. */
2992         thread_ip_inc(p);
2993 }
2994
2995 static inline void
2996 instr_mov_s_exec(struct rte_swx_pipeline *p)
2997 {
2998         struct thread *t = &p->threads[p->thread_id];
2999         struct instruction *ip = t->ip;
3000
3001         TRACE("[Thread %2u] mov (s)\n",
3002               p->thread_id);
3003
3004         MOV_S(t, ip);
3005
3006         /* Thread. */
3007         thread_ip_inc(p);
3008 }
3009
3010 static inline void
3011 instr_mov_i_exec(struct rte_swx_pipeline *p)
3012 {
3013         struct thread *t = &p->threads[p->thread_id];
3014         struct instruction *ip = t->ip;
3015
3016         TRACE("[Thread %2u] mov m.f %x\n",
3017               p->thread_id,
3018               ip->mov.src_val);
3019
3020         MOV_I(t, ip);
3021
3022         /* Thread. */
3023         thread_ip_inc(p);
3024 }
3025
3026 /*
3027  * dma.
3028  */
3029 static int
3030 instr_dma_translate(struct rte_swx_pipeline *p,
3031                     struct action *action,
3032                     char **tokens,
3033                     int n_tokens,
3034                     struct instruction *instr,
3035                     struct instruction_data *data __rte_unused)
3036 {
3037         char *dst = tokens[1];
3038         char *src = tokens[2];
3039         struct header *h;
3040         struct field *tf;
3041
3042         CHECK(action, EINVAL);
3043         CHECK(n_tokens == 3, EINVAL);
3044
3045         h = header_parse(p, dst);
3046         CHECK(h, EINVAL);
3047
3048         tf = action_field_parse(action, src);
3049         CHECK(tf, EINVAL);
3050
3051         instr->type = INSTR_DMA_HT;
3052         instr->dma.dst.header_id[0] = h->id;
3053         instr->dma.dst.struct_id[0] = h->struct_id;
3054         instr->dma.n_bytes[0] = h->st->n_bits / 8;
3055         instr->dma.src.offset[0] = tf->offset / 8;
3056
3057         return 0;
3058 }
3059
3060 static inline void
3061 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3062
3063 static inline void
3064 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3065 {
3066         struct thread *t = &p->threads[p->thread_id];
3067         struct instruction *ip = t->ip;
3068         uint8_t *action_data = t->structs[0];
3069         uint64_t valid_headers = t->valid_headers;
3070         uint32_t i;
3071
3072         for (i = 0; i < n_dma; i++) {
3073                 uint32_t header_id = ip->dma.dst.header_id[i];
3074                 uint32_t struct_id = ip->dma.dst.struct_id[i];
3075                 uint32_t offset = ip->dma.src.offset[i];
3076                 uint32_t n_bytes = ip->dma.n_bytes[i];
3077
3078                 struct header_runtime *h = &t->headers[header_id];
3079                 uint8_t *h_ptr0 = h->ptr0;
3080                 uint8_t *h_ptr = t->structs[struct_id];
3081
3082                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3083                         h_ptr : h_ptr0;
3084                 void *src = &action_data[offset];
3085
3086                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3087
3088                 /* Headers. */
3089                 memcpy(dst, src, n_bytes);
3090                 t->structs[struct_id] = dst;
3091                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3092         }
3093
3094         t->valid_headers = valid_headers;
3095 }
3096
3097 static inline void
3098 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3099 {
3100         __instr_dma_ht_exec(p, 1);
3101
3102         /* Thread. */
3103         thread_ip_inc(p);
3104 }
3105
3106 static inline void
3107 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3108 {
3109         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3110               p->thread_id);
3111
3112         __instr_dma_ht_exec(p, 2);
3113
3114         /* Thread. */
3115         thread_ip_inc(p);
3116 }
3117
3118 static inline void
3119 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3120 {
3121         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3122               p->thread_id);
3123
3124         __instr_dma_ht_exec(p, 3);
3125
3126         /* Thread. */
3127         thread_ip_inc(p);
3128 }
3129
3130 static inline void
3131 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3132 {
3133         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3134               p->thread_id);
3135
3136         __instr_dma_ht_exec(p, 4);
3137
3138         /* Thread. */
3139         thread_ip_inc(p);
3140 }
3141
3142 static inline void
3143 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3144 {
3145         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3146               p->thread_id);
3147
3148         __instr_dma_ht_exec(p, 5);
3149
3150         /* Thread. */
3151         thread_ip_inc(p);
3152 }
3153
3154 static inline void
3155 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3156 {
3157         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3158               p->thread_id);
3159
3160         __instr_dma_ht_exec(p, 6);
3161
3162         /* Thread. */
3163         thread_ip_inc(p);
3164 }
3165
3166 static inline void
3167 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3168 {
3169         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3170               p->thread_id);
3171
3172         __instr_dma_ht_exec(p, 7);
3173
3174         /* Thread. */
3175         thread_ip_inc(p);
3176 }
3177
3178 static inline void
3179 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3180 {
3181         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3182               p->thread_id);
3183
3184         __instr_dma_ht_exec(p, 8);
3185
3186         /* Thread. */
3187         thread_ip_inc(p);
3188 }
3189
3190 /*
3191  * alu.
3192  */
3193 static int
3194 instr_alu_add_translate(struct rte_swx_pipeline *p,
3195                         struct action *action,
3196                         char **tokens,
3197                         int n_tokens,
3198                         struct instruction *instr,
3199                         struct instruction_data *data __rte_unused)
3200 {
3201         char *dst = tokens[1], *src = tokens[2];
3202         struct field *fdst, *fsrc;
3203         uint32_t dst_struct_id, src_struct_id, src_val;
3204
3205         CHECK(n_tokens == 3, EINVAL);
3206
3207         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3208         CHECK(fdst, EINVAL);
3209
3210         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3211         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3212         if (fsrc) {
3213                 instr->type = INSTR_ALU_ADD;
3214                 if (dst[0] == 'h' && src[0] == 'm')
3215                         instr->type = INSTR_ALU_ADD_HM;
3216                 if (dst[0] == 'm' && src[0] == 'h')
3217                         instr->type = INSTR_ALU_ADD_MH;
3218                 if (dst[0] == 'h' && src[0] == 'h')
3219                         instr->type = INSTR_ALU_ADD_HH;
3220
3221                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3222                 instr->alu.dst.n_bits = fdst->n_bits;
3223                 instr->alu.dst.offset = fdst->offset / 8;
3224                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3225                 instr->alu.src.n_bits = fsrc->n_bits;
3226                 instr->alu.src.offset = fsrc->offset / 8;
3227                 return 0;
3228         }
3229
3230         /* ADD_MI, ADD_HI. */
3231         src_val = strtoul(src, &src, 0);
3232         CHECK(!src[0], EINVAL);
3233
3234         instr->type = INSTR_ALU_ADD_MI;
3235         if (dst[0] == 'h')
3236                 instr->type = INSTR_ALU_ADD_HI;
3237
3238         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3239         instr->alu.dst.n_bits = fdst->n_bits;
3240         instr->alu.dst.offset = fdst->offset / 8;
3241         instr->alu.src_val = (uint32_t)src_val;
3242         return 0;
3243 }
3244
3245 static int
3246 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3247                         struct action *action,
3248                         char **tokens,
3249                         int n_tokens,
3250                         struct instruction *instr,
3251                         struct instruction_data *data __rte_unused)
3252 {
3253         char *dst = tokens[1], *src = tokens[2];
3254         struct field *fdst, *fsrc;
3255         uint32_t dst_struct_id, src_struct_id, src_val;
3256
3257         CHECK(n_tokens == 3, EINVAL);
3258
3259         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3260         CHECK(fdst, EINVAL);
3261
3262         /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3263         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3264         if (fsrc) {
3265                 instr->type = INSTR_ALU_SUB;
3266                 if (dst[0] == 'h' && src[0] == 'm')
3267                         instr->type = INSTR_ALU_SUB_HM;
3268                 if (dst[0] == 'm' && src[0] == 'h')
3269                         instr->type = INSTR_ALU_SUB_MH;
3270                 if (dst[0] == 'h' && src[0] == 'h')
3271                         instr->type = INSTR_ALU_SUB_HH;
3272
3273                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3274                 instr->alu.dst.n_bits = fdst->n_bits;
3275                 instr->alu.dst.offset = fdst->offset / 8;
3276                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3277                 instr->alu.src.n_bits = fsrc->n_bits;
3278                 instr->alu.src.offset = fsrc->offset / 8;
3279                 return 0;
3280         }
3281
3282         /* SUB_MI, SUB_HI. */
3283         src_val = strtoul(src, &src, 0);
3284         CHECK(!src[0], EINVAL);
3285
3286         instr->type = INSTR_ALU_SUB_MI;
3287         if (dst[0] == 'h')
3288                 instr->type = INSTR_ALU_SUB_HI;
3289
3290         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3291         instr->alu.dst.n_bits = fdst->n_bits;
3292         instr->alu.dst.offset = fdst->offset / 8;
3293         instr->alu.src_val = (uint32_t)src_val;
3294         return 0;
3295 }
3296
3297 static int
3298 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3299                           struct action *action __rte_unused,
3300                           char **tokens,
3301                           int n_tokens,
3302                           struct instruction *instr,
3303                           struct instruction_data *data __rte_unused)
3304 {
3305         char *dst = tokens[1], *src = tokens[2];
3306         struct header *hdst, *hsrc;
3307         struct field *fdst, *fsrc;
3308
3309         CHECK(n_tokens == 3, EINVAL);
3310
3311         fdst = header_field_parse(p, dst, &hdst);
3312         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3313
3314         /* CKADD_FIELD. */
3315         fsrc = header_field_parse(p, src, &hsrc);
3316         if (fsrc) {
3317                 instr->type = INSTR_ALU_CKADD_FIELD;
3318                 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3319                 instr->alu.dst.n_bits = fdst->n_bits;
3320                 instr->alu.dst.offset = fdst->offset / 8;
3321                 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3322                 instr->alu.src.n_bits = fsrc->n_bits;
3323                 instr->alu.src.offset = fsrc->offset / 8;
3324                 return 0;
3325         }
3326
3327         /* CKADD_STRUCT, CKADD_STRUCT20. */
3328         hsrc = header_parse(p, src);
3329         CHECK(hsrc, EINVAL);
3330
3331         instr->type = INSTR_ALU_CKADD_STRUCT;
3332         if ((hsrc->st->n_bits / 8) == 20)
3333                 instr->type = INSTR_ALU_CKADD_STRUCT20;
3334
3335         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3336         instr->alu.dst.n_bits = fdst->n_bits;
3337         instr->alu.dst.offset = fdst->offset / 8;
3338         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3339         instr->alu.src.n_bits = hsrc->st->n_bits;
3340         instr->alu.src.offset = 0; /* Unused. */
3341         return 0;
3342 }
3343
3344 static int
3345 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3346                           struct action *action __rte_unused,
3347                           char **tokens,
3348                           int n_tokens,
3349                           struct instruction *instr,
3350                           struct instruction_data *data __rte_unused)
3351 {
3352         char *dst = tokens[1], *src = tokens[2];
3353         struct header *hdst, *hsrc;
3354         struct field *fdst, *fsrc;
3355
3356         CHECK(n_tokens == 3, EINVAL);
3357
3358         fdst = header_field_parse(p, dst, &hdst);
3359         CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3360
3361         fsrc = header_field_parse(p, src, &hsrc);
3362         CHECK(fsrc, EINVAL);
3363
3364         instr->type = INSTR_ALU_CKSUB_FIELD;
3365         instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3366         instr->alu.dst.n_bits = fdst->n_bits;
3367         instr->alu.dst.offset = fdst->offset / 8;
3368         instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3369         instr->alu.src.n_bits = fsrc->n_bits;
3370         instr->alu.src.offset = fsrc->offset / 8;
3371         return 0;
3372 }
3373
3374 static int
3375 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3376                         struct action *action,
3377                         char **tokens,
3378                         int n_tokens,
3379                         struct instruction *instr,
3380                         struct instruction_data *data __rte_unused)
3381 {
3382         char *dst = tokens[1], *src = tokens[2];
3383         struct field *fdst, *fsrc;
3384         uint32_t dst_struct_id, src_struct_id, src_val;
3385
3386         CHECK(n_tokens == 3, EINVAL);
3387
3388         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3389         CHECK(fdst, EINVAL);
3390
3391         /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3392         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3393         if (fsrc) {
3394                 instr->type = INSTR_ALU_SHL;
3395                 if (dst[0] == 'h' && src[0] == 'm')
3396                         instr->type = INSTR_ALU_SHL_HM;
3397                 if (dst[0] == 'm' && src[0] == 'h')
3398                         instr->type = INSTR_ALU_SHL_MH;
3399                 if (dst[0] == 'h' && src[0] == 'h')
3400                         instr->type = INSTR_ALU_SHL_HH;
3401
3402                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3403                 instr->alu.dst.n_bits = fdst->n_bits;
3404                 instr->alu.dst.offset = fdst->offset / 8;
3405                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3406                 instr->alu.src.n_bits = fsrc->n_bits;
3407                 instr->alu.src.offset = fsrc->offset / 8;
3408                 return 0;
3409         }
3410
3411         /* SHL_MI, SHL_HI. */
3412         src_val = strtoul(src, &src, 0);
3413         CHECK(!src[0], EINVAL);
3414
3415         instr->type = INSTR_ALU_SHL_MI;
3416         if (dst[0] == 'h')
3417                 instr->type = INSTR_ALU_SHL_HI;
3418
3419         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3420         instr->alu.dst.n_bits = fdst->n_bits;
3421         instr->alu.dst.offset = fdst->offset / 8;
3422         instr->alu.src_val = (uint32_t)src_val;
3423         return 0;
3424 }
3425
3426 static int
3427 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3428                         struct action *action,
3429                         char **tokens,
3430                         int n_tokens,
3431                         struct instruction *instr,
3432                         struct instruction_data *data __rte_unused)
3433 {
3434         char *dst = tokens[1], *src = tokens[2];
3435         struct field *fdst, *fsrc;
3436         uint32_t dst_struct_id, src_struct_id, src_val;
3437
3438         CHECK(n_tokens == 3, EINVAL);
3439
3440         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3441         CHECK(fdst, EINVAL);
3442
3443         /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3444         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3445         if (fsrc) {
3446                 instr->type = INSTR_ALU_SHR;
3447                 if (dst[0] == 'h' && src[0] == 'm')
3448                         instr->type = INSTR_ALU_SHR_HM;
3449                 if (dst[0] == 'm' && src[0] == 'h')
3450                         instr->type = INSTR_ALU_SHR_MH;
3451                 if (dst[0] == 'h' && src[0] == 'h')
3452                         instr->type = INSTR_ALU_SHR_HH;
3453
3454                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3455                 instr->alu.dst.n_bits = fdst->n_bits;
3456                 instr->alu.dst.offset = fdst->offset / 8;
3457                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3458                 instr->alu.src.n_bits = fsrc->n_bits;
3459                 instr->alu.src.offset = fsrc->offset / 8;
3460                 return 0;
3461         }
3462
3463         /* SHR_MI, SHR_HI. */
3464         src_val = strtoul(src, &src, 0);
3465         CHECK(!src[0], EINVAL);
3466
3467         instr->type = INSTR_ALU_SHR_MI;
3468         if (dst[0] == 'h')
3469                 instr->type = INSTR_ALU_SHR_HI;
3470
3471         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3472         instr->alu.dst.n_bits = fdst->n_bits;
3473         instr->alu.dst.offset = fdst->offset / 8;
3474         instr->alu.src_val = (uint32_t)src_val;
3475         return 0;
3476 }
3477
3478 static int
3479 instr_alu_and_translate(struct rte_swx_pipeline *p,
3480                         struct action *action,
3481                         char **tokens,
3482                         int n_tokens,
3483                         struct instruction *instr,
3484                         struct instruction_data *data __rte_unused)
3485 {
3486         char *dst = tokens[1], *src = tokens[2];
3487         struct field *fdst, *fsrc;
3488         uint32_t dst_struct_id, src_struct_id, src_val;
3489
3490         CHECK(n_tokens == 3, EINVAL);
3491
3492         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3493         CHECK(fdst, EINVAL);
3494
3495         /* AND or AND_S. */
3496         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3497         if (fsrc) {
3498                 instr->type = INSTR_ALU_AND;
3499                 if ((dst[0] == 'h' && src[0] != 'h') ||
3500                     (dst[0] != 'h' && src[0] == 'h'))
3501                         instr->type = INSTR_ALU_AND_S;
3502
3503                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3504                 instr->alu.dst.n_bits = fdst->n_bits;
3505                 instr->alu.dst.offset = fdst->offset / 8;
3506                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3507                 instr->alu.src.n_bits = fsrc->n_bits;
3508                 instr->alu.src.offset = fsrc->offset / 8;
3509                 return 0;
3510         }
3511
3512         /* AND_I. */
3513         src_val = strtoul(src, &src, 0);
3514         CHECK(!src[0], EINVAL);
3515
3516         if (dst[0] == 'h')
3517                 src_val = htonl(src_val);
3518
3519         instr->type = INSTR_ALU_AND_I;
3520         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3521         instr->alu.dst.n_bits = fdst->n_bits;
3522         instr->alu.dst.offset = fdst->offset / 8;
3523         instr->alu.src_val = (uint32_t)src_val;
3524         return 0;
3525 }
3526
3527 static int
3528 instr_alu_or_translate(struct rte_swx_pipeline *p,
3529                        struct action *action,
3530                        char **tokens,
3531                        int n_tokens,
3532                        struct instruction *instr,
3533                        struct instruction_data *data __rte_unused)
3534 {
3535         char *dst = tokens[1], *src = tokens[2];
3536         struct field *fdst, *fsrc;
3537         uint32_t dst_struct_id, src_struct_id, src_val;
3538
3539         CHECK(n_tokens == 3, EINVAL);
3540
3541         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3542         CHECK(fdst, EINVAL);
3543
3544         /* OR or OR_S. */
3545         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3546         if (fsrc) {
3547                 instr->type = INSTR_ALU_OR;
3548                 if ((dst[0] == 'h' && src[0] != 'h') ||
3549                     (dst[0] != 'h' && src[0] == 'h'))
3550                         instr->type = INSTR_ALU_OR_S;
3551
3552                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3553                 instr->alu.dst.n_bits = fdst->n_bits;
3554                 instr->alu.dst.offset = fdst->offset / 8;
3555                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3556                 instr->alu.src.n_bits = fsrc->n_bits;
3557                 instr->alu.src.offset = fsrc->offset / 8;
3558                 return 0;
3559         }
3560
3561         /* OR_I. */
3562         src_val = strtoul(src, &src, 0);
3563         CHECK(!src[0], EINVAL);
3564
3565         if (dst[0] == 'h')
3566                 src_val = htonl(src_val);
3567
3568         instr->type = INSTR_ALU_OR_I;
3569         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3570         instr->alu.dst.n_bits = fdst->n_bits;
3571         instr->alu.dst.offset = fdst->offset / 8;
3572         instr->alu.src_val = (uint32_t)src_val;
3573         return 0;
3574 }
3575
3576 static int
3577 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3578                         struct action *action,
3579                         char **tokens,
3580                         int n_tokens,
3581                         struct instruction *instr,
3582                         struct instruction_data *data __rte_unused)
3583 {
3584         char *dst = tokens[1], *src = tokens[2];
3585         struct field *fdst, *fsrc;
3586         uint32_t dst_struct_id, src_struct_id, src_val;
3587
3588         CHECK(n_tokens == 3, EINVAL);
3589
3590         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3591         CHECK(fdst, EINVAL);
3592
3593         /* XOR or XOR_S. */
3594         fsrc = struct_field_parse(p, action, src, &src_struct_id);
3595         if (fsrc) {
3596                 instr->type = INSTR_ALU_XOR;
3597                 if ((dst[0] == 'h' && src[0] != 'h') ||
3598                     (dst[0] != 'h' && src[0] == 'h'))
3599                         instr->type = INSTR_ALU_XOR_S;
3600
3601                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3602                 instr->alu.dst.n_bits = fdst->n_bits;
3603                 instr->alu.dst.offset = fdst->offset / 8;
3604                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3605                 instr->alu.src.n_bits = fsrc->n_bits;
3606                 instr->alu.src.offset = fsrc->offset / 8;
3607                 return 0;
3608         }
3609
3610         /* XOR_I. */
3611         src_val = strtoul(src, &src, 0);
3612         CHECK(!src[0], EINVAL);
3613
3614         if (dst[0] == 'h')
3615                 src_val = htonl(src_val);
3616
3617         instr->type = INSTR_ALU_XOR_I;
3618         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3619         instr->alu.dst.n_bits = fdst->n_bits;
3620         instr->alu.dst.offset = fdst->offset / 8;
3621         instr->alu.src_val = (uint32_t)src_val;
3622         return 0;
3623 }
3624
3625 static inline void
3626 instr_alu_add_exec(struct rte_swx_pipeline *p)
3627 {
3628         struct thread *t = &p->threads[p->thread_id];
3629         struct instruction *ip = t->ip;
3630
3631         TRACE("[Thread %2u] add\n", p->thread_id);
3632
3633         /* Structs. */
3634         ALU(t, ip, +);
3635
3636         /* Thread. */
3637         thread_ip_inc(p);
3638 }
3639
3640 static inline void
3641 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3642 {
3643         struct thread *t = &p->threads[p->thread_id];
3644         struct instruction *ip = t->ip;
3645
3646         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
3647
3648         /* Structs. */
3649         ALU_MH(t, ip, +);
3650
3651         /* Thread. */
3652         thread_ip_inc(p);
3653 }
3654
3655 static inline void
3656 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3657 {
3658         struct thread *t = &p->threads[p->thread_id];
3659         struct instruction *ip = t->ip;
3660
3661         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
3662
3663         /* Structs. */
3664         ALU_HM(t, ip, +);
3665
3666         /* Thread. */
3667         thread_ip_inc(p);
3668 }
3669
3670 static inline void
3671 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3672 {
3673         struct thread *t = &p->threads[p->thread_id];
3674         struct instruction *ip = t->ip;
3675
3676         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
3677
3678         /* Structs. */
3679         ALU_HH(t, ip, +);
3680
3681         /* Thread. */
3682         thread_ip_inc(p);
3683 }
3684
3685 static inline void
3686 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3687 {
3688         struct thread *t = &p->threads[p->thread_id];
3689         struct instruction *ip = t->ip;
3690
3691         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
3692
3693         /* Structs. */
3694         ALU_MI(t, ip, +);
3695
3696         /* Thread. */
3697         thread_ip_inc(p);
3698 }
3699
3700 static inline void
3701 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3702 {
3703         struct thread *t = &p->threads[p->thread_id];
3704         struct instruction *ip = t->ip;
3705
3706         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3707
3708         /* Structs. */
3709         ALU_HI(t, ip, +);
3710
3711         /* Thread. */
3712         thread_ip_inc(p);
3713 }
3714
3715 static inline void
3716 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3717 {
3718         struct thread *t = &p->threads[p->thread_id];
3719         struct instruction *ip = t->ip;
3720
3721         TRACE("[Thread %2u] sub\n", p->thread_id);
3722
3723         /* Structs. */
3724         ALU(t, ip, -);
3725
3726         /* Thread. */
3727         thread_ip_inc(p);
3728 }
3729
3730 static inline void
3731 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3732 {
3733         struct thread *t = &p->threads[p->thread_id];
3734         struct instruction *ip = t->ip;
3735
3736         TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
3737
3738         /* Structs. */
3739         ALU_MH(t, ip, -);
3740
3741         /* Thread. */
3742         thread_ip_inc(p);
3743 }
3744
3745 static inline void
3746 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3747 {
3748         struct thread *t = &p->threads[p->thread_id];
3749         struct instruction *ip = t->ip;
3750
3751         TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
3752
3753         /* Structs. */
3754         ALU_HM(t, ip, -);
3755
3756         /* Thread. */
3757         thread_ip_inc(p);
3758 }
3759
3760 static inline void
3761 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3762 {
3763         struct thread *t = &p->threads[p->thread_id];
3764         struct instruction *ip = t->ip;
3765
3766         TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
3767
3768         /* Structs. */
3769         ALU_HH(t, ip, -);
3770
3771         /* Thread. */
3772         thread_ip_inc(p);
3773 }
3774
3775 static inline void
3776 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3777 {
3778         struct thread *t = &p->threads[p->thread_id];
3779         struct instruction *ip = t->ip;
3780
3781         TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
3782
3783         /* Structs. */
3784         ALU_MI(t, ip, -);
3785
3786         /* Thread. */
3787         thread_ip_inc(p);
3788 }
3789
3790 static inline void
3791 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3792 {
3793         struct thread *t = &p->threads[p->thread_id];
3794         struct instruction *ip = t->ip;
3795
3796         TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
3797
3798         /* Structs. */
3799         ALU_HI(t, ip, -);
3800
3801         /* Thread. */
3802         thread_ip_inc(p);
3803 }
3804
3805 static inline void
3806 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3807 {
3808         struct thread *t = &p->threads[p->thread_id];
3809         struct instruction *ip = t->ip;
3810
3811         TRACE("[Thread %2u] shl\n", p->thread_id);
3812
3813         /* Structs. */
3814         ALU(t, ip, <<);
3815
3816         /* Thread. */
3817         thread_ip_inc(p);
3818 }
3819
3820 static inline void
3821 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3822 {
3823         struct thread *t = &p->threads[p->thread_id];
3824         struct instruction *ip = t->ip;
3825
3826         TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
3827
3828         /* Structs. */
3829         ALU_MH(t, ip, <<);
3830
3831         /* Thread. */
3832         thread_ip_inc(p);
3833 }
3834
3835 static inline void
3836 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3837 {
3838         struct thread *t = &p->threads[p->thread_id];
3839         struct instruction *ip = t->ip;
3840
3841         TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
3842
3843         /* Structs. */
3844         ALU_HM(t, ip, <<);
3845
3846         /* Thread. */
3847         thread_ip_inc(p);
3848 }
3849
3850 static inline void
3851 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3852 {
3853         struct thread *t = &p->threads[p->thread_id];
3854         struct instruction *ip = t->ip;
3855
3856         TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
3857
3858         /* Structs. */
3859         ALU_HH(t, ip, <<);
3860
3861         /* Thread. */
3862         thread_ip_inc(p);
3863 }
3864
3865 static inline void
3866 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3867 {
3868         struct thread *t = &p->threads[p->thread_id];
3869         struct instruction *ip = t->ip;
3870
3871         TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
3872
3873         /* Structs. */
3874         ALU_MI(t, ip, <<);
3875
3876         /* Thread. */
3877         thread_ip_inc(p);
3878 }
3879
3880 static inline void
3881 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3882 {
3883         struct thread *t = &p->threads[p->thread_id];
3884         struct instruction *ip = t->ip;
3885
3886         TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
3887
3888         /* Structs. */
3889         ALU_HI(t, ip, <<);
3890
3891         /* Thread. */
3892         thread_ip_inc(p);
3893 }
3894
3895 static inline void
3896 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3897 {
3898         struct thread *t = &p->threads[p->thread_id];
3899         struct instruction *ip = t->ip;
3900
3901         TRACE("[Thread %2u] shr\n", p->thread_id);
3902
3903         /* Structs. */
3904         ALU(t, ip, >>);
3905
3906         /* Thread. */
3907         thread_ip_inc(p);
3908 }
3909
3910 static inline void
3911 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3912 {
3913         struct thread *t = &p->threads[p->thread_id];
3914         struct instruction *ip = t->ip;
3915
3916         TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
3917
3918         /* Structs. */
3919         ALU_MH(t, ip, >>);
3920
3921         /* Thread. */
3922         thread_ip_inc(p);
3923 }
3924
3925 static inline void
3926 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3927 {
3928         struct thread *t = &p->threads[p->thread_id];
3929         struct instruction *ip = t->ip;
3930
3931         TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
3932
3933         /* Structs. */
3934         ALU_HM(t, ip, >>);
3935
3936         /* Thread. */
3937         thread_ip_inc(p);
3938 }
3939
3940 static inline void
3941 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3942 {
3943         struct thread *t = &p->threads[p->thread_id];
3944         struct instruction *ip = t->ip;
3945
3946         TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
3947
3948         /* Structs. */
3949         ALU_HH(t, ip, >>);
3950
3951         /* Thread. */
3952         thread_ip_inc(p);
3953 }
3954
3955 static inline void
3956 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3957 {
3958         struct thread *t = &p->threads[p->thread_id];
3959         struct instruction *ip = t->ip;
3960
3961         TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
3962
3963         /* Structs. */
3964         ALU_MI(t, ip, >>);
3965
3966         /* Thread. */
3967         thread_ip_inc(p);
3968 }
3969
3970 static inline void
3971 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3972 {
3973         struct thread *t = &p->threads[p->thread_id];
3974         struct instruction *ip = t->ip;
3975
3976         TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
3977
3978         /* Structs. */
3979         ALU_HI(t, ip, >>);
3980
3981         /* Thread. */
3982         thread_ip_inc(p);
3983 }
3984
3985 static inline void
3986 instr_alu_and_exec(struct rte_swx_pipeline *p)
3987 {
3988         struct thread *t = &p->threads[p->thread_id];
3989         struct instruction *ip = t->ip;
3990
3991         TRACE("[Thread %2u] and\n", p->thread_id);
3992
3993         /* Structs. */
3994         ALU(t, ip, &);
3995
3996         /* Thread. */
3997         thread_ip_inc(p);
3998 }
3999
4000 static inline void
4001 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
4002 {
4003         struct thread *t = &p->threads[p->thread_id];
4004         struct instruction *ip = t->ip;
4005
4006         TRACE("[Thread %2u] and (s)\n", p->thread_id);
4007
4008         /* Structs. */
4009         ALU_S(t, ip, &);
4010
4011         /* Thread. */
4012         thread_ip_inc(p);
4013 }
4014
4015 static inline void
4016 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4017 {
4018         struct thread *t = &p->threads[p->thread_id];
4019         struct instruction *ip = t->ip;
4020
4021         TRACE("[Thread %2u] and (i)\n", p->thread_id);
4022
4023         /* Structs. */
4024         ALU_I(t, ip, &);
4025
4026         /* Thread. */
4027         thread_ip_inc(p);
4028 }
4029
4030 static inline void
4031 instr_alu_or_exec(struct rte_swx_pipeline *p)
4032 {
4033         struct thread *t = &p->threads[p->thread_id];
4034         struct instruction *ip = t->ip;
4035
4036         TRACE("[Thread %2u] or\n", p->thread_id);
4037
4038         /* Structs. */
4039         ALU(t, ip, |);
4040
4041         /* Thread. */
4042         thread_ip_inc(p);
4043 }
4044
4045 static inline void
4046 instr_alu_or_s_exec(struct rte_swx_pipeline *p)
4047 {
4048         struct thread *t = &p->threads[p->thread_id];
4049         struct instruction *ip = t->ip;
4050
4051         TRACE("[Thread %2u] or (s)\n", p->thread_id);
4052
4053         /* Structs. */
4054         ALU_S(t, ip, |);
4055
4056         /* Thread. */
4057         thread_ip_inc(p);
4058 }
4059
4060 static inline void
4061 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4062 {
4063         struct thread *t = &p->threads[p->thread_id];
4064         struct instruction *ip = t->ip;
4065
4066         TRACE("[Thread %2u] or (i)\n", p->thread_id);
4067
4068         /* Structs. */
4069         ALU_I(t, ip, |);
4070
4071         /* Thread. */
4072         thread_ip_inc(p);
4073 }
4074
4075 static inline void
4076 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4077 {
4078         struct thread *t = &p->threads[p->thread_id];
4079         struct instruction *ip = t->ip;
4080
4081         TRACE("[Thread %2u] xor\n", p->thread_id);
4082
4083         /* Structs. */
4084         ALU(t, ip, ^);
4085
4086         /* Thread. */
4087         thread_ip_inc(p);
4088 }
4089
4090 static inline void
4091 instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
4092 {
4093         struct thread *t = &p->threads[p->thread_id];
4094         struct instruction *ip = t->ip;
4095
4096         TRACE("[Thread %2u] xor (s)\n", p->thread_id);
4097
4098         /* Structs. */
4099         ALU_S(t, ip, ^);
4100
4101         /* Thread. */
4102         thread_ip_inc(p);
4103 }
4104
4105 static inline void
4106 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4107 {
4108         struct thread *t = &p->threads[p->thread_id];
4109         struct instruction *ip = t->ip;
4110
4111         TRACE("[Thread %2u] xor (i)\n", p->thread_id);
4112
4113         /* Structs. */
4114         ALU_I(t, ip, ^);
4115
4116         /* Thread. */
4117         thread_ip_inc(p);
4118 }
4119
4120 static inline void
4121 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4122 {
4123         struct thread *t = &p->threads[p->thread_id];
4124         struct instruction *ip = t->ip;
4125         uint8_t *dst_struct, *src_struct;
4126         uint16_t *dst16_ptr, dst;
4127         uint64_t *src64_ptr, src64, src64_mask, src;
4128         uint64_t r;
4129
4130         TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
4131
4132         /* Structs. */
4133         dst_struct = t->structs[ip->alu.dst.struct_id];
4134         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4135         dst = *dst16_ptr;
4136
4137         src_struct = t->structs[ip->alu.src.struct_id];
4138         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4139         src64 = *src64_ptr;
4140         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4141         src = src64 & src64_mask;
4142
4143         r = dst;
4144         r = ~r & 0xFFFF;
4145
4146         /* The first input (r) is a 16-bit number. The second and the third
4147          * inputs are 32-bit numbers. In the worst case scenario, the sum of the
4148          * three numbers (output r) is a 34-bit number.
4149          */
4150         r += (src >> 32) + (src & 0xFFFFFFFF);
4151
4152         /* The first input is a 16-bit number. The second input is an 18-bit
4153          * number. In the worst case scenario, the sum of the two numbers is a
4154          * 19-bit number.
4155          */
4156         r = (r & 0xFFFF) + (r >> 16);
4157
4158         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4159          * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
4160          */
4161         r = (r & 0xFFFF) + (r >> 16);
4162
4163         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4164          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4165          * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
4166          * therefore the output r is always a 16-bit number.
4167          */
4168         r = (r & 0xFFFF) + (r >> 16);
4169
4170         r = ~r & 0xFFFF;
4171         r = r ? r : 0xFFFF;
4172
4173         *dst16_ptr = (uint16_t)r;
4174
4175         /* Thread. */
4176         thread_ip_inc(p);
4177 }
4178
4179 static inline void
4180 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4181 {
4182         struct thread *t = &p->threads[p->thread_id];
4183         struct instruction *ip = t->ip;
4184         uint8_t *dst_struct, *src_struct;
4185         uint16_t *dst16_ptr, dst;
4186         uint64_t *src64_ptr, src64, src64_mask, src;
4187         uint64_t r;
4188
4189         TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
4190
4191         /* Structs. */
4192         dst_struct = t->structs[ip->alu.dst.struct_id];
4193         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4194         dst = *dst16_ptr;
4195
4196         src_struct = t->structs[ip->alu.src.struct_id];
4197         src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4198         src64 = *src64_ptr;
4199         src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4200         src = src64 & src64_mask;
4201
4202         r = dst;
4203         r = ~r & 0xFFFF;
4204
4205         /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
4206          * the following sequence of operations in 2's complement arithmetic:
4207          *    a '- b = (a - b) % 0xFFFF.
4208          *
4209          * In order to prevent an underflow for the below subtraction, in which
4210          * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
4211          * minuend), we first add a multiple of the 0xFFFF modulus to the
4212          * minuend. The number we add to the minuend needs to be a 34-bit number
4213          * or higher, so for readability reasons we picked the 36-bit multiple.
4214          * We are effectively turning the 16-bit minuend into a 36-bit number:
4215          *    (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
4216          */
4217         r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
4218
4219         /* A 33-bit number is subtracted from a 36-bit number (the input r). The
4220          * result (the output r) is a 36-bit number.
4221          */
4222         r -= (src >> 32) + (src & 0xFFFFFFFF);
4223
4224         /* The first input is a 16-bit number. The second input is a 20-bit
4225          * number. Their sum is a 21-bit number.
4226          */
4227         r = (r & 0xFFFF) + (r >> 16);
4228
4229         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4230          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
4231          */
4232         r = (r & 0xFFFF) + (r >> 16);
4233
4234         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4235          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4236          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4237          * generated, therefore the output r is always a 16-bit number.
4238          */
4239         r = (r & 0xFFFF) + (r >> 16);
4240
4241         r = ~r & 0xFFFF;
4242         r = r ? r : 0xFFFF;
4243
4244         *dst16_ptr = (uint16_t)r;
4245
4246         /* Thread. */
4247         thread_ip_inc(p);
4248 }
4249
4250 static inline void
4251 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4252 {
4253         struct thread *t = &p->threads[p->thread_id];
4254         struct instruction *ip = t->ip;
4255         uint8_t *dst_struct, *src_struct;
4256         uint16_t *dst16_ptr;
4257         uint32_t *src32_ptr;
4258         uint64_t r0, r1;
4259
4260         TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
4261
4262         /* Structs. */
4263         dst_struct = t->structs[ip->alu.dst.struct_id];
4264         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4265
4266         src_struct = t->structs[ip->alu.src.struct_id];
4267         src32_ptr = (uint32_t *)&src_struct[0];
4268
4269         r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
4270         r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
4271         r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
4272         r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
4273         r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
4274
4275         /* The first input is a 16-bit number. The second input is a 19-bit
4276          * number. Their sum is a 20-bit number.
4277          */
4278         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4279
4280         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4281          * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
4282          */
4283         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4284
4285         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4286          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4287          * 0x1000E), the output r is (0 .. 15). So no carry bit can be
4288          * generated, therefore the output r is always a 16-bit number.
4289          */
4290         r0 = (r0 & 0xFFFF) + (r0 >> 16);
4291
4292         r0 = ~r0 & 0xFFFF;
4293         r0 = r0 ? r0 : 0xFFFF;
4294
4295         *dst16_ptr = (uint16_t)r0;
4296
4297         /* Thread. */
4298         thread_ip_inc(p);
4299 }
4300
4301 static inline void
4302 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4303 {
4304         struct thread *t = &p->threads[p->thread_id];
4305         struct instruction *ip = t->ip;
4306         uint8_t *dst_struct, *src_struct;
4307         uint16_t *dst16_ptr;
4308         uint32_t *src32_ptr;
4309         uint64_t r = 0;
4310         uint32_t i;
4311
4312         TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
4313
4314         /* Structs. */
4315         dst_struct = t->structs[ip->alu.dst.struct_id];
4316         dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4317
4318         src_struct = t->structs[ip->alu.src.struct_id];
4319         src32_ptr = (uint32_t *)&src_struct[0];
4320
4321         /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
4322          * Therefore, in the worst case scenario, a 35-bit number is added to a
4323          * 16-bit number (the input r), so the output r is 36-bit number.
4324          */
4325         for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
4326                 r += *src32_ptr;
4327
4328         /* The first input is a 16-bit number. The second input is a 20-bit
4329          * number. Their sum is a 21-bit number.
4330          */
4331         r = (r & 0xFFFF) + (r >> 16);
4332
4333         /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4334          * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
4335          */
4336         r = (r & 0xFFFF) + (r >> 16);
4337
4338         /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4339          * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4340          * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4341          * generated, therefore the output r is always a 16-bit number.
4342          */
4343         r = (r & 0xFFFF) + (r >> 16);
4344
4345         r = ~r & 0xFFFF;
4346         r = r ? r : 0xFFFF;
4347
4348         *dst16_ptr = (uint16_t)r;
4349
4350         /* Thread. */
4351         thread_ip_inc(p);
4352 }
4353
4354 #define RTE_SWX_INSTRUCTION_TOKENS_MAX 16
4355
4356 static int
4357 instr_translate(struct rte_swx_pipeline *p,
4358                 struct action *action,
4359                 char *string,
4360                 struct instruction *instr,
4361                 struct instruction_data *data)
4362 {
4363         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
4364         int n_tokens = 0, tpos = 0;
4365
4366         /* Parse the instruction string into tokens. */
4367         for ( ; ; ) {
4368                 char *token;
4369
4370                 token = strtok_r(string, " \t\v", &string);
4371                 if (!token)
4372                         break;
4373
4374                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
4375
4376                 tokens[n_tokens] = token;
4377                 n_tokens++;
4378         }
4379
4380         CHECK(n_tokens, EINVAL);
4381
4382         /* Handle the optional instruction label. */
4383         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
4384                 strcpy(data->label, tokens[0]);
4385
4386                 tpos += 2;
4387                 CHECK(n_tokens - tpos, EINVAL);
4388         }
4389
4390         /* Identify the instruction type. */
4391         if (!strcmp(tokens[tpos], "rx"))
4392                 return instr_rx_translate(p,
4393                                           action,
4394                                           &tokens[tpos],
4395                                           n_tokens - tpos,
4396                                           instr,
4397                                           data);
4398
4399         if (!strcmp(tokens[tpos], "tx"))
4400                 return instr_tx_translate(p,
4401                                           action,
4402                                           &tokens[tpos],
4403                                           n_tokens - tpos,
4404                                           instr,
4405                                           data);
4406
4407         if (!strcmp(tokens[tpos], "extract"))
4408                 return instr_hdr_extract_translate(p,
4409                                                    action,
4410                                                    &tokens[tpos],
4411                                                    n_tokens - tpos,
4412                                                    instr,
4413                                                    data);
4414
4415         if (!strcmp(tokens[tpos], "emit"))
4416                 return instr_hdr_emit_translate(p,
4417                                                 action,
4418                                                 &tokens[tpos],
4419                                                 n_tokens - tpos,
4420                                                 instr,
4421                                                 data);
4422
4423         if (!strcmp(tokens[tpos], "validate"))
4424                 return instr_hdr_validate_translate(p,
4425                                                     action,
4426                                                     &tokens[tpos],
4427                                                     n_tokens - tpos,
4428                                                     instr,
4429                                                     data);
4430
4431         if (!strcmp(tokens[tpos], "invalidate"))
4432                 return instr_hdr_invalidate_translate(p,
4433                                                       action,
4434                                                       &tokens[tpos],
4435                                                       n_tokens - tpos,
4436                                                       instr,
4437                                                       data);
4438
4439         if (!strcmp(tokens[tpos], "mov"))
4440                 return instr_mov_translate(p,
4441                                            action,
4442                                            &tokens[tpos],
4443                                            n_tokens - tpos,
4444                                            instr,
4445                                            data);
4446
4447         if (!strcmp(tokens[tpos], "dma"))
4448                 return instr_dma_translate(p,
4449                                            action,
4450                                            &tokens[tpos],
4451                                            n_tokens - tpos,
4452                                            instr,
4453                                            data);
4454
4455         if (!strcmp(tokens[tpos], "add"))
4456                 return instr_alu_add_translate(p,
4457                                                action,
4458                                                &tokens[tpos],
4459                                                n_tokens - tpos,
4460                                                instr,
4461                                                data);
4462
4463         if (!strcmp(tokens[tpos], "sub"))
4464                 return instr_alu_sub_translate(p,
4465                                                action,
4466                                                &tokens[tpos],
4467                                                n_tokens - tpos,
4468                                                instr,
4469                                                data);
4470
4471         if (!strcmp(tokens[tpos], "ckadd"))
4472                 return instr_alu_ckadd_translate(p,
4473                                                  action,
4474                                                  &tokens[tpos],
4475                                                  n_tokens - tpos,
4476                                                  instr,
4477                                                  data);
4478
4479         if (!strcmp(tokens[tpos], "cksub"))
4480                 return instr_alu_cksub_translate(p,
4481                                                  action,
4482                                                  &tokens[tpos],
4483                                                  n_tokens - tpos,
4484                                                  instr,
4485                                                  data);
4486
4487         if (!strcmp(tokens[tpos], "and"))
4488                 return instr_alu_and_translate(p,
4489                                                action,
4490                                                &tokens[tpos],
4491                                                n_tokens - tpos,
4492                                                instr,
4493                                                data);
4494
4495         if (!strcmp(tokens[tpos], "or"))
4496                 return instr_alu_or_translate(p,
4497                                               action,
4498                                               &tokens[tpos],
4499                                               n_tokens - tpos,
4500                                               instr,
4501                                               data);
4502
4503         if (!strcmp(tokens[tpos], "xor"))
4504                 return instr_alu_xor_translate(p,
4505                                                action,
4506                                                &tokens[tpos],
4507                                                n_tokens - tpos,
4508                                                instr,
4509                                                data);
4510
4511         if (!strcmp(tokens[tpos], "shl"))
4512                 return instr_alu_shl_translate(p,
4513                                                action,
4514                                                &tokens[tpos],
4515                                                n_tokens - tpos,
4516                                                instr,
4517                                                data);
4518
4519         if (!strcmp(tokens[tpos], "shr"))
4520                 return instr_alu_shr_translate(p,
4521                                                action,
4522                                                &tokens[tpos],
4523                                                n_tokens - tpos,
4524                                                instr,
4525                                                data);
4526
4527         if (!strcmp(tokens[tpos], "table"))
4528                 return instr_table_translate(p,
4529                                              action,
4530                                              &tokens[tpos],
4531                                              n_tokens - tpos,
4532                                              instr,
4533                                              data);
4534
4535         if (!strcmp(tokens[tpos], "extern"))
4536                 return instr_extern_translate(p,
4537                                               action,
4538                                               &tokens[tpos],
4539                                               n_tokens - tpos,
4540                                               instr,
4541                                               data);
4542
4543         CHECK(0, EINVAL);
4544 }
4545
4546 static uint32_t
4547 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
4548 {
4549         uint32_t count = 0, i;
4550
4551         if (!label[0])
4552                 return 0;
4553
4554         for (i = 0; i < n; i++)
4555                 if (!strcmp(label, data[i].jmp_label))
4556                         count++;
4557
4558         return count;
4559 }
4560
4561 static int
4562 instr_label_check(struct instruction_data *instruction_data,
4563                   uint32_t n_instructions)
4564 {
4565         uint32_t i;
4566
4567         /* Check that all instruction labels are unique. */
4568         for (i = 0; i < n_instructions; i++) {
4569                 struct instruction_data *data = &instruction_data[i];
4570                 char *label = data->label;
4571                 uint32_t j;
4572
4573                 if (!label[0])
4574                         continue;
4575
4576                 for (j = i + 1; j < n_instructions; j++)
4577                         CHECK(strcmp(label, data[j].label), EINVAL);
4578         }
4579
4580         /* Get users for each instruction label. */
4581         for (i = 0; i < n_instructions; i++) {
4582                 struct instruction_data *data = &instruction_data[i];
4583                 char *label = data->label;
4584
4585                 data->n_users = label_is_used(instruction_data,
4586                                               n_instructions,
4587                                               label);
4588         }
4589
4590         return 0;
4591 }
4592
4593 static int
4594 instruction_config(struct rte_swx_pipeline *p,
4595                    struct action *a,
4596                    const char **instructions,
4597                    uint32_t n_instructions)
4598 {
4599         struct instruction *instr = NULL;
4600         struct instruction_data *data = NULL;
4601         char *string = NULL;
4602         int err = 0;
4603         uint32_t i;
4604
4605         CHECK(n_instructions, EINVAL);
4606         CHECK(instructions, EINVAL);
4607         for (i = 0; i < n_instructions; i++)
4608                 CHECK(instructions[i], EINVAL);
4609
4610         /* Memory allocation. */
4611         instr = calloc(n_instructions, sizeof(struct instruction));
4612         if (!instr) {
4613                 err = ENOMEM;
4614                 goto error;
4615         }
4616
4617         data = calloc(n_instructions, sizeof(struct instruction_data));
4618         if (!data) {
4619                 err = ENOMEM;
4620                 goto error;
4621         }
4622
4623         for (i = 0; i < n_instructions; i++) {
4624                 string = strdup(instructions[i]);
4625                 if (!string) {
4626                         err = ENOMEM;
4627                         goto error;
4628                 }
4629
4630                 err = instr_translate(p, a, string, &instr[i], &data[i]);
4631                 if (err)
4632                         goto error;
4633
4634                 free(string);
4635         }
4636
4637         err = instr_label_check(data, n_instructions);
4638         if (err)
4639                 goto error;
4640
4641         free(data);
4642
4643         if (a) {
4644                 a->instructions = instr;
4645                 a->n_instructions = n_instructions;
4646         } else {
4647                 p->instructions = instr;
4648                 p->n_instructions = n_instructions;
4649         }
4650
4651         return 0;
4652
4653 error:
4654         free(string);
4655         free(data);
4656         free(instr);
4657         return err;
4658 }
4659
4660 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
4661
4662 static instr_exec_t instruction_table[] = {
4663         [INSTR_RX] = instr_rx_exec,
4664         [INSTR_TX] = instr_tx_exec,
4665
4666         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
4667         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
4668         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
4669         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
4670         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
4671         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
4672         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
4673         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
4674
4675         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
4676         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
4677         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
4678         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
4679         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
4680         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
4681         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
4682         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
4683         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
4684
4685         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
4686         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
4687
4688         [INSTR_MOV] = instr_mov_exec,
4689         [INSTR_MOV_S] = instr_mov_s_exec,
4690         [INSTR_MOV_I] = instr_mov_i_exec,
4691
4692         [INSTR_DMA_HT] = instr_dma_ht_exec,
4693         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
4694         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
4695         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
4696         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
4697         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
4698         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
4699         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
4700
4701         [INSTR_ALU_ADD] = instr_alu_add_exec,
4702         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
4703         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
4704         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
4705         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
4706         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
4707
4708         [INSTR_ALU_SUB] = instr_alu_sub_exec,
4709         [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
4710         [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
4711         [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
4712         [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
4713         [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
4714
4715         [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
4716         [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
4717         [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
4718         [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
4719
4720         [INSTR_ALU_AND] = instr_alu_and_exec,
4721         [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
4722         [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
4723
4724         [INSTR_ALU_OR] = instr_alu_or_exec,
4725         [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
4726         [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
4727
4728         [INSTR_ALU_XOR] = instr_alu_xor_exec,
4729         [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec,
4730         [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
4731
4732         [INSTR_ALU_SHL] = instr_alu_shl_exec,
4733         [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
4734         [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
4735         [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
4736         [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
4737         [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
4738
4739         [INSTR_ALU_SHR] = instr_alu_shr_exec,
4740         [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
4741         [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
4742         [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
4743         [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
4744         [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
4745
4746         [INSTR_TABLE] = instr_table_exec,
4747         [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
4748         [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
4749 };
4750
4751 static inline void
4752 instr_exec(struct rte_swx_pipeline *p)
4753 {
4754         struct thread *t = &p->threads[p->thread_id];
4755         struct instruction *ip = t->ip;
4756         instr_exec_t instr = instruction_table[ip->type];
4757
4758         instr(p);
4759 }
4760
4761 /*
4762  * Action.
4763  */
4764 static struct action *
4765 action_find(struct rte_swx_pipeline *p, const char *name)
4766 {
4767         struct action *elem;
4768
4769         if (!name)
4770                 return NULL;
4771
4772         TAILQ_FOREACH(elem, &p->actions, node)
4773                 if (strcmp(elem->name, name) == 0)
4774                         return elem;
4775
4776         return NULL;
4777 }
4778
4779 static struct field *
4780 action_field_find(struct action *a, const char *name)
4781 {
4782         return a->st ? struct_type_field_find(a->st, name) : NULL;
4783 }
4784
4785 static struct field *
4786 action_field_parse(struct action *action, const char *name)
4787 {
4788         if (name[0] != 't' || name[1] != '.')
4789                 return NULL;
4790
4791         return action_field_find(action, &name[2]);
4792 }
4793
4794 int
4795 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
4796                                const char *name,
4797                                const char *args_struct_type_name,
4798                                const char **instructions,
4799                                uint32_t n_instructions)
4800 {
4801         struct struct_type *args_struct_type;
4802         struct action *a;
4803         int err;
4804
4805         CHECK(p, EINVAL);
4806
4807         CHECK_NAME(name, EINVAL);
4808         CHECK(!action_find(p, name), EEXIST);
4809
4810         if (args_struct_type_name) {
4811                 CHECK_NAME(args_struct_type_name, EINVAL);
4812                 args_struct_type = struct_type_find(p, args_struct_type_name);
4813                 CHECK(args_struct_type, EINVAL);
4814         } else {
4815                 args_struct_type = NULL;
4816         }
4817
4818         /* Node allocation. */
4819         a = calloc(1, sizeof(struct action));
4820         CHECK(a, ENOMEM);
4821
4822         /* Node initialization. */
4823         strcpy(a->name, name);
4824         a->st = args_struct_type;
4825         a->id = p->n_actions;
4826
4827         /* Instruction translation. */
4828         err = instruction_config(p, a, instructions, n_instructions);
4829         if (err) {
4830                 free(a);
4831                 return err;
4832         }
4833
4834         /* Node add to tailq. */
4835         TAILQ_INSERT_TAIL(&p->actions, a, node);
4836         p->n_actions++;
4837
4838         return 0;
4839 }
4840
4841 static int
4842 action_build(struct rte_swx_pipeline *p)
4843 {
4844         struct action *action;
4845
4846         p->action_instructions = calloc(p->n_actions,
4847                                         sizeof(struct instruction *));
4848         CHECK(p->action_instructions, ENOMEM);
4849
4850         TAILQ_FOREACH(action, &p->actions, node)
4851                 p->action_instructions[action->id] = action->instructions;
4852
4853         return 0;
4854 }
4855
4856 static void
4857 action_build_free(struct rte_swx_pipeline *p)
4858 {
4859         free(p->action_instructions);
4860         p->action_instructions = NULL;
4861 }
4862
4863 static void
4864 action_free(struct rte_swx_pipeline *p)
4865 {
4866         action_build_free(p);
4867
4868         for ( ; ; ) {
4869                 struct action *action;
4870
4871                 action = TAILQ_FIRST(&p->actions);
4872                 if (!action)
4873                         break;
4874
4875                 TAILQ_REMOVE(&p->actions, action, node);
4876                 free(action->instructions);
4877                 free(action);
4878         }
4879 }
4880
4881 /*
4882  * Table.
4883  */
4884 static struct table_type *
4885 table_type_find(struct rte_swx_pipeline *p, const char *name)
4886 {
4887         struct table_type *elem;
4888
4889         TAILQ_FOREACH(elem, &p->table_types, node)
4890                 if (strcmp(elem->name, name) == 0)
4891                         return elem;
4892
4893         return NULL;
4894 }
4895
4896 static struct table_type *
4897 table_type_resolve(struct rte_swx_pipeline *p,
4898                    const char *recommended_type_name,
4899                    enum rte_swx_table_match_type match_type)
4900 {
4901         struct table_type *elem;
4902
4903         /* Only consider the recommended type if the match type is correct. */
4904         if (recommended_type_name)
4905                 TAILQ_FOREACH(elem, &p->table_types, node)
4906                         if (!strcmp(elem->name, recommended_type_name) &&
4907                             (elem->match_type == match_type))
4908                                 return elem;
4909
4910         /* Ignore the recommended type and get the first element with this match
4911          * type.
4912          */
4913         TAILQ_FOREACH(elem, &p->table_types, node)
4914                 if (elem->match_type == match_type)
4915                         return elem;
4916
4917         return NULL;
4918 }
4919
4920 static struct table *
4921 table_find(struct rte_swx_pipeline *p, const char *name)
4922 {
4923         struct table *elem;
4924
4925         TAILQ_FOREACH(elem, &p->tables, node)
4926                 if (strcmp(elem->name, name) == 0)
4927                         return elem;
4928
4929         return NULL;
4930 }
4931
4932 static struct table *
4933 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
4934 {
4935         struct table *table = NULL;
4936
4937         TAILQ_FOREACH(table, &p->tables, node)
4938                 if (table->id == id)
4939                         return table;
4940
4941         return NULL;
4942 }
4943
4944 int
4945 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
4946                                      const char *name,
4947                                      enum rte_swx_table_match_type match_type,
4948                                      struct rte_swx_table_ops *ops)
4949 {
4950         struct table_type *elem;
4951
4952         CHECK(p, EINVAL);
4953
4954         CHECK_NAME(name, EINVAL);
4955         CHECK(!table_type_find(p, name), EEXIST);
4956
4957         CHECK(ops, EINVAL);
4958         CHECK(ops->create, EINVAL);
4959         CHECK(ops->lkp, EINVAL);
4960         CHECK(ops->free, EINVAL);
4961
4962         /* Node allocation. */
4963         elem = calloc(1, sizeof(struct table_type));
4964         CHECK(elem, ENOMEM);
4965
4966         /* Node initialization. */
4967         strcpy(elem->name, name);
4968         elem->match_type = match_type;
4969         memcpy(&elem->ops, ops, sizeof(*ops));
4970
4971         /* Node add to tailq. */
4972         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
4973
4974         return 0;
4975 }
4976
4977 static enum rte_swx_table_match_type
4978 table_match_type_resolve(struct rte_swx_match_field_params *fields,
4979                          uint32_t n_fields)
4980 {
4981         uint32_t i;
4982
4983         for (i = 0; i < n_fields; i++)
4984                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
4985                         break;
4986
4987         if (i == n_fields)
4988                 return RTE_SWX_TABLE_MATCH_EXACT;
4989
4990         if ((i == n_fields - 1) &&
4991             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
4992                 return RTE_SWX_TABLE_MATCH_LPM;
4993
4994         return RTE_SWX_TABLE_MATCH_WILDCARD;
4995 }
4996
4997 int
4998 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
4999                               const char *name,
5000                               struct rte_swx_pipeline_table_params *params,
5001                               const char *recommended_table_type_name,
5002                               const char *args,
5003                               uint32_t size)
5004 {
5005         struct table_type *type;
5006         struct table *t;
5007         struct action *default_action;
5008         struct header *header = NULL;
5009         int is_header = 0;
5010         uint32_t offset_prev = 0, action_data_size_max = 0, i;
5011
5012         CHECK(p, EINVAL);
5013
5014         CHECK_NAME(name, EINVAL);
5015         CHECK(!table_find(p, name), EEXIST);
5016
5017         CHECK(params, EINVAL);
5018
5019         /* Match checks. */
5020         CHECK(!params->n_fields || params->fields, EINVAL);
5021         for (i = 0; i < params->n_fields; i++) {
5022                 struct rte_swx_match_field_params *field = &params->fields[i];
5023                 struct header *h;
5024                 struct field *hf, *mf;
5025                 uint32_t offset;
5026
5027                 CHECK_NAME(field->name, EINVAL);
5028
5029                 hf = header_field_parse(p, field->name, &h);
5030                 mf = metadata_field_parse(p, field->name);
5031                 CHECK(hf || mf, EINVAL);
5032
5033                 offset = hf ? hf->offset : mf->offset;
5034
5035                 if (i == 0) {
5036                         is_header = hf ? 1 : 0;
5037                         header = hf ? h : NULL;
5038                         offset_prev = offset;
5039
5040                         continue;
5041                 }
5042
5043                 CHECK((is_header && hf && (h->id == header->id)) ||
5044                       (!is_header && mf), EINVAL);
5045
5046                 CHECK(offset > offset_prev, EINVAL);
5047                 offset_prev = offset;
5048         }
5049
5050         /* Action checks. */
5051         CHECK(params->n_actions, EINVAL);
5052         CHECK(params->action_names, EINVAL);
5053         for (i = 0; i < params->n_actions; i++) {
5054                 const char *action_name = params->action_names[i];
5055                 struct action *a;
5056                 uint32_t action_data_size;
5057
5058                 CHECK(action_name, EINVAL);
5059
5060                 a = action_find(p, action_name);
5061                 CHECK(a, EINVAL);
5062
5063                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
5064                 if (action_data_size > action_data_size_max)
5065                         action_data_size_max = action_data_size;
5066         }
5067
5068         CHECK(params->default_action_name, EINVAL);
5069         for (i = 0; i < p->n_actions; i++)
5070                 if (!strcmp(params->action_names[i],
5071                             params->default_action_name))
5072                         break;
5073         CHECK(i < params->n_actions, EINVAL);
5074         default_action = action_find(p, params->default_action_name);
5075         CHECK((default_action->st && params->default_action_data) ||
5076               !params->default_action_data, EINVAL);
5077
5078         /* Table type checks. */
5079         if (params->n_fields) {
5080                 enum rte_swx_table_match_type match_type;
5081
5082                 match_type = table_match_type_resolve(params->fields,
5083                                                       params->n_fields);
5084                 type = table_type_resolve(p,
5085                                           recommended_table_type_name,
5086                                           match_type);
5087                 CHECK(type, EINVAL);
5088         } else {
5089                 type = NULL;
5090         }
5091
5092         /* Memory allocation. */
5093         t = calloc(1, sizeof(struct table));
5094         CHECK(t, ENOMEM);
5095
5096         t->fields = calloc(params->n_fields, sizeof(struct match_field));
5097         if (!t->fields) {
5098                 free(t);
5099                 CHECK(0, ENOMEM);
5100         }
5101
5102         t->actions = calloc(params->n_actions, sizeof(struct action *));
5103         if (!t->actions) {
5104                 free(t->fields);
5105                 free(t);
5106                 CHECK(0, ENOMEM);
5107         }
5108
5109         if (action_data_size_max) {
5110                 t->default_action_data = calloc(1, action_data_size_max);
5111                 if (!t->default_action_data) {
5112                         free(t->actions);
5113                         free(t->fields);
5114                         free(t);
5115                         CHECK(0, ENOMEM);
5116                 }
5117         }
5118
5119         /* Node initialization. */
5120         strcpy(t->name, name);
5121         if (args && args[0])
5122                 strcpy(t->args, args);
5123         t->type = type;
5124
5125         for (i = 0; i < params->n_fields; i++) {
5126                 struct rte_swx_match_field_params *field = &params->fields[i];
5127                 struct match_field *f = &t->fields[i];
5128
5129                 f->match_type = field->match_type;
5130                 f->field = is_header ?
5131                         header_field_parse(p, field->name, NULL) :
5132                         metadata_field_parse(p, field->name);
5133         }
5134         t->n_fields = params->n_fields;
5135         t->is_header = is_header;
5136         t->header = header;
5137
5138         for (i = 0; i < params->n_actions; i++)
5139                 t->actions[i] = action_find(p, params->action_names[i]);
5140         t->default_action = default_action;
5141         if (default_action->st)
5142                 memcpy(t->default_action_data,
5143                        params->default_action_data,
5144                        default_action->st->n_bits / 8);
5145         t->n_actions = params->n_actions;
5146         t->default_action_is_const = params->default_action_is_const;
5147         t->action_data_size_max = action_data_size_max;
5148
5149         t->size = size;
5150         t->id = p->n_tables;
5151
5152         /* Node add to tailq. */
5153         TAILQ_INSERT_TAIL(&p->tables, t, node);
5154         p->n_tables++;
5155
5156         return 0;
5157 }
5158
5159 static struct rte_swx_table_params *
5160 table_params_get(struct table *table)
5161 {
5162         struct rte_swx_table_params *params;
5163         struct field *first, *last;
5164         uint8_t *key_mask;
5165         uint32_t key_size, key_offset, action_data_size, i;
5166
5167         /* Memory allocation. */
5168         params = calloc(1, sizeof(struct rte_swx_table_params));
5169         if (!params)
5170                 return NULL;
5171
5172         /* Key offset and size. */
5173         first = table->fields[0].field;
5174         last = table->fields[table->n_fields - 1].field;
5175         key_offset = first->offset / 8;
5176         key_size = (last->offset + last->n_bits - first->offset) / 8;
5177
5178         /* Memory allocation. */
5179         key_mask = calloc(1, key_size);
5180         if (!key_mask) {
5181                 free(params);
5182                 return NULL;
5183         }
5184
5185         /* Key mask. */
5186         for (i = 0; i < table->n_fields; i++) {
5187                 struct field *f = table->fields[i].field;
5188                 uint32_t start = (f->offset - first->offset) / 8;
5189                 size_t size = f->n_bits / 8;
5190
5191                 memset(&key_mask[start], 0xFF, size);
5192         }
5193
5194         /* Action data size. */
5195         action_data_size = 0;
5196         for (i = 0; i < table->n_actions; i++) {
5197                 struct action *action = table->actions[i];
5198                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
5199
5200                 if (ads > action_data_size)
5201                         action_data_size = ads;
5202         }
5203
5204         /* Fill in. */
5205         params->match_type = table->type->match_type;
5206         params->key_size = key_size;
5207         params->key_offset = key_offset;
5208         params->key_mask0 = key_mask;
5209         params->action_data_size = action_data_size;
5210         params->n_keys_max = table->size;
5211
5212         return params;
5213 }
5214
5215 static void
5216 table_params_free(struct rte_swx_table_params *params)
5217 {
5218         if (!params)
5219                 return;
5220
5221         free(params->key_mask0);
5222         free(params);
5223 }
5224
5225 static int
5226 table_state_build(struct rte_swx_pipeline *p)
5227 {
5228         struct table *table;
5229
5230         p->table_state = calloc(p->n_tables,
5231                                 sizeof(struct rte_swx_table_state));
5232         CHECK(p->table_state, ENOMEM);
5233
5234         TAILQ_FOREACH(table, &p->tables, node) {
5235                 struct rte_swx_table_state *ts = &p->table_state[table->id];
5236
5237                 if (table->type) {
5238                         struct rte_swx_table_params *params;
5239
5240                         /* ts->obj. */
5241                         params = table_params_get(table);
5242                         CHECK(params, ENOMEM);
5243
5244                         ts->obj = table->type->ops.create(params,
5245                                 NULL,
5246                                 table->args,
5247                                 p->numa_node);
5248
5249                         table_params_free(params);
5250                         CHECK(ts->obj, ENODEV);
5251                 }
5252
5253                 /* ts->default_action_data. */
5254                 if (table->action_data_size_max) {
5255                         ts->default_action_data =
5256                                 malloc(table->action_data_size_max);
5257                         CHECK(ts->default_action_data, ENOMEM);
5258
5259                         memcpy(ts->default_action_data,
5260                                table->default_action_data,
5261                                table->action_data_size_max);
5262                 }
5263
5264                 /* ts->default_action_id. */
5265                 ts->default_action_id = table->default_action->id;
5266         }
5267
5268         return 0;
5269 }
5270
5271 static void
5272 table_state_build_free(struct rte_swx_pipeline *p)
5273 {
5274         uint32_t i;
5275
5276         if (!p->table_state)
5277                 return;
5278
5279         for (i = 0; i < p->n_tables; i++) {
5280                 struct rte_swx_table_state *ts = &p->table_state[i];
5281                 struct table *table = table_find_by_id(p, i);
5282
5283                 /* ts->obj. */
5284                 if (table->type && ts->obj)
5285                         table->type->ops.free(ts->obj);
5286
5287                 /* ts->default_action_data. */
5288                 free(ts->default_action_data);
5289         }
5290
5291         free(p->table_state);
5292         p->table_state = NULL;
5293 }
5294
5295 static void
5296 table_state_free(struct rte_swx_pipeline *p)
5297 {
5298         table_state_build_free(p);
5299 }
5300
5301 static int
5302 table_stub_lkp(void *table __rte_unused,
5303                void *mailbox __rte_unused,
5304                uint8_t **key __rte_unused,
5305                uint64_t *action_id __rte_unused,
5306                uint8_t **action_data __rte_unused,
5307                int *hit)
5308 {
5309         *hit = 0;
5310         return 1; /* DONE. */
5311 }
5312
5313 static int
5314 table_build(struct rte_swx_pipeline *p)
5315 {
5316         uint32_t i;
5317
5318         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
5319                 struct thread *t = &p->threads[i];
5320                 struct table *table;
5321
5322                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
5323                 CHECK(t->tables, ENOMEM);
5324
5325                 TAILQ_FOREACH(table, &p->tables, node) {
5326                         struct table_runtime *r = &t->tables[table->id];
5327
5328                         if (table->type) {
5329                                 uint64_t size;
5330
5331                                 size = table->type->ops.mailbox_size_get();
5332
5333                                 /* r->func. */
5334                                 r->func = table->type->ops.lkp;
5335
5336                                 /* r->mailbox. */
5337                                 if (size) {
5338                                         r->mailbox = calloc(1, size);
5339                                         CHECK(r->mailbox, ENOMEM);
5340                                 }
5341
5342                                 /* r->key. */
5343                                 r->key = table->is_header ?
5344                                         &t->structs[table->header->struct_id] :
5345                                         &t->structs[p->metadata_struct_id];
5346                         } else {
5347                                 r->func = table_stub_lkp;
5348                         }
5349                 }
5350         }
5351
5352         return 0;
5353 }
5354
5355 static void
5356 table_build_free(struct rte_swx_pipeline *p)
5357 {
5358         uint32_t i;
5359
5360         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
5361                 struct thread *t = &p->threads[i];
5362                 uint32_t j;
5363
5364                 if (!t->tables)
5365                         continue;
5366
5367                 for (j = 0; j < p->n_tables; j++) {
5368                         struct table_runtime *r = &t->tables[j];
5369
5370                         free(r->mailbox);
5371                 }
5372
5373                 free(t->tables);
5374                 t->tables = NULL;
5375         }
5376 }
5377
5378 static void
5379 table_free(struct rte_swx_pipeline *p)
5380 {
5381         table_build_free(p);
5382
5383         /* Tables. */
5384         for ( ; ; ) {
5385                 struct table *elem;
5386
5387                 elem = TAILQ_FIRST(&p->tables);
5388                 if (!elem)
5389                         break;
5390
5391                 TAILQ_REMOVE(&p->tables, elem, node);
5392                 free(elem->fields);
5393                 free(elem->actions);
5394                 free(elem->default_action_data);
5395                 free(elem);
5396         }
5397
5398         /* Table types. */
5399         for ( ; ; ) {
5400                 struct table_type *elem;
5401
5402                 elem = TAILQ_FIRST(&p->table_types);
5403                 if (!elem)
5404                         break;
5405
5406                 TAILQ_REMOVE(&p->table_types, elem, node);
5407                 free(elem);
5408         }
5409 }
5410
5411 /*
5412  * Pipeline.
5413  */
5414 int
5415 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
5416 {
5417         struct rte_swx_pipeline *pipeline;
5418
5419         /* Check input parameters. */
5420         CHECK(p, EINVAL);
5421
5422         /* Memory allocation. */
5423         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
5424         CHECK(pipeline, ENOMEM);
5425
5426         /* Initialization. */
5427         TAILQ_INIT(&pipeline->struct_types);
5428         TAILQ_INIT(&pipeline->port_in_types);
5429         TAILQ_INIT(&pipeline->ports_in);
5430         TAILQ_INIT(&pipeline->port_out_types);
5431         TAILQ_INIT(&pipeline->ports_out);
5432         TAILQ_INIT(&pipeline->extern_types);
5433         TAILQ_INIT(&pipeline->extern_objs);
5434         TAILQ_INIT(&pipeline->extern_funcs);
5435         TAILQ_INIT(&pipeline->headers);
5436         TAILQ_INIT(&pipeline->actions);
5437         TAILQ_INIT(&pipeline->table_types);
5438         TAILQ_INIT(&pipeline->tables);
5439
5440         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
5441         pipeline->numa_node = numa_node;
5442
5443         *p = pipeline;
5444         return 0;
5445 }
5446
5447 void
5448 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
5449 {
5450         if (!p)
5451                 return;
5452
5453         free(p->instructions);
5454
5455         table_state_free(p);
5456         table_free(p);
5457         action_free(p);
5458         metadata_free(p);
5459         header_free(p);
5460         extern_func_free(p);
5461         extern_obj_free(p);
5462         port_out_free(p);
5463         port_in_free(p);
5464         struct_free(p);
5465
5466         free(p);
5467 }
5468
5469 int
5470 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
5471                                      const char **instructions,
5472                                      uint32_t n_instructions)
5473 {
5474         int err;
5475         uint32_t i;
5476
5477         err = instruction_config(p, NULL, instructions, n_instructions);
5478         if (err)
5479                 return err;
5480
5481         /* Thread instruction pointer reset. */
5482         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
5483                 struct thread *t = &p->threads[i];
5484
5485                 thread_ip_reset(p, t);
5486         }
5487
5488         return 0;
5489 }
5490
5491 int
5492 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
5493 {
5494         int status;
5495
5496         CHECK(p, EINVAL);
5497         CHECK(p->build_done == 0, EEXIST);
5498
5499         status = port_in_build(p);
5500         if (status)
5501                 goto error;
5502
5503         status = port_out_build(p);
5504         if (status)
5505                 goto error;
5506
5507         status = struct_build(p);
5508         if (status)
5509                 goto error;
5510
5511         status = extern_obj_build(p);
5512         if (status)
5513                 goto error;
5514
5515         status = extern_func_build(p);
5516         if (status)
5517                 goto error;
5518
5519         status = header_build(p);
5520         if (status)
5521                 goto error;
5522
5523         status = metadata_build(p);
5524         if (status)
5525                 goto error;
5526
5527         status = action_build(p);
5528         if (status)
5529                 goto error;
5530
5531         status = table_build(p);
5532         if (status)
5533                 goto error;
5534
5535         status = table_state_build(p);
5536         if (status)
5537                 goto error;
5538
5539         p->build_done = 1;
5540         return 0;
5541
5542 error:
5543         table_state_build_free(p);
5544         table_build_free(p);
5545         action_build_free(p);
5546         metadata_build_free(p);
5547         header_build_free(p);
5548         extern_func_build_free(p);
5549         extern_obj_build_free(p);
5550         port_out_build_free(p);
5551         port_in_build_free(p);
5552         struct_build_free(p);
5553
5554         return status;
5555 }
5556
5557 void
5558 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
5559 {
5560         uint32_t i;
5561
5562         for (i = 0; i < n_instructions; i++)
5563                 instr_exec(p);
5564 }
5565
5566 /*
5567  * Control.
5568  */
5569 int
5570 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
5571                                  struct rte_swx_table_state **table_state)
5572 {
5573         if (!p || !table_state || !p->build_done)
5574                 return -EINVAL;
5575
5576         *table_state = p->table_state;
5577         return 0;
5578 }
5579
5580 int
5581 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
5582                                  struct rte_swx_table_state *table_state)
5583 {
5584         if (!p || !table_state || !p->build_done)
5585                 return -EINVAL;
5586
5587         p->table_state = table_state;
5588         return 0;
5589 }