lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9 #include <arpa/inet.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_prefetch.h>
  13 #include <rte_byteorder.h>
  14
  15 #include "rte_swx_pipeline.h"
  16 #include "rte_swx_ctl.h"
  17
  18 #define CHECK(condition, err_code)                                             \
  19 do {                                                                           \
  20         if (!(condition))                                                      \
  21                 return -(err_code);                                            \
  22 } while (0)
  23
  24 #define CHECK_NAME(name, err_code)                                             \
  25         CHECK((name) && (name)[0], err_code)
  26
  27 #ifndef TRACE_LEVEL
  28 #define TRACE_LEVEL 0
  29 #endif
  30
  31 #if TRACE_LEVEL
  32 #define TRACE(...) printf(__VA_ARGS__)
  33 #else
  34 #define TRACE(...)
  35 #endif
  36
  37 #define ntoh64(x) rte_be_to_cpu_64(x)
  38 #define hton64(x) rte_cpu_to_be_64(x)
  39
  40 /*
  41  * Struct.
  42  */
  43 struct field {
  44         char name[RTE_SWX_NAME_SIZE];
  45         uint32_t n_bits;
  46         uint32_t offset;
  47 };
  48
  49 struct struct_type {
  50         TAILQ_ENTRY(struct_type) node;
  51         char name[RTE_SWX_NAME_SIZE];
  52         struct field *fields;
  53         uint32_t n_fields;
  54         uint32_t n_bits;
  55 };
  56
  57 TAILQ_HEAD(struct_type_tailq, struct_type);
  58
  59 /*
  60  * Input port.
  61  */
  62 struct port_in_type {
  63         TAILQ_ENTRY(port_in_type) node;
  64         char name[RTE_SWX_NAME_SIZE];
  65         struct rte_swx_port_in_ops ops;
  66 };
  67
  68 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  69
  70 struct port_in {
  71         TAILQ_ENTRY(port_in) node;
  72         struct port_in_type *type;
  73         void *obj;
  74         uint32_t id;
  75 };
  76
  77 TAILQ_HEAD(port_in_tailq, port_in);
  78
  79 struct port_in_runtime {
  80         rte_swx_port_in_pkt_rx_t pkt_rx;
  81         void *obj;
  82 };
  83
  84 /*
  85  * Output port.
  86  */
  87 struct port_out_type {
  88         TAILQ_ENTRY(port_out_type) node;
  89         char name[RTE_SWX_NAME_SIZE];
  90         struct rte_swx_port_out_ops ops;
  91 };
  92
  93 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  94
  95 struct port_out {
  96         TAILQ_ENTRY(port_out) node;
  97         struct port_out_type *type;
  98         void *obj;
  99         uint32_t id;
 100 };
 101
 102 TAILQ_HEAD(port_out_tailq, port_out);
 103
 104 struct port_out_runtime {
 105         rte_swx_port_out_pkt_tx_t pkt_tx;
 106         rte_swx_port_out_flush_t flush;
 107         void *obj;
 108 };
 109
 110 /*
 111  * Extern object.
 112  */
 113 struct extern_type_member_func {
 114         TAILQ_ENTRY(extern_type_member_func) node;
 115         char name[RTE_SWX_NAME_SIZE];
 116         rte_swx_extern_type_member_func_t func;
 117         uint32_t id;
 118 };
 119
 120 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 121
 122 struct extern_type {
 123         TAILQ_ENTRY(extern_type) node;
 124         char name[RTE_SWX_NAME_SIZE];
 125         struct struct_type *mailbox_struct_type;
 126         rte_swx_extern_type_constructor_t constructor;
 127         rte_swx_extern_type_destructor_t destructor;
 128         struct extern_type_member_func_tailq funcs;
 129         uint32_t n_funcs;
 130 };
 131
 132 TAILQ_HEAD(extern_type_tailq, extern_type);
 133
 134 struct extern_obj {
 135         TAILQ_ENTRY(extern_obj) node;
 136         char name[RTE_SWX_NAME_SIZE];
 137         struct extern_type *type;
 138         void *obj;
 139         uint32_t struct_id;
 140         uint32_t id;
 141 };
 142
 143 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 144
 145 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 146 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 147 #endif
 148
 149 struct extern_obj_runtime {
 150         void *obj;
 151         uint8_t *mailbox;
 152         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 153 };
 154
 155 /*
 156  * Extern function.
 157  */
 158 struct extern_func {
 159         TAILQ_ENTRY(extern_func) node;
 160         char name[RTE_SWX_NAME_SIZE];
 161         struct struct_type *mailbox_struct_type;
 162         rte_swx_extern_func_t func;
 163         uint32_t struct_id;
 164         uint32_t id;
 165 };
 166
 167 TAILQ_HEAD(extern_func_tailq, extern_func);
 168
 169 struct extern_func_runtime {
 170         uint8_t *mailbox;
 171         rte_swx_extern_func_t func;
 172 };
 173
 174 /*
 175  * Header.
 176  */
 177 struct header {
 178         TAILQ_ENTRY(header) node;
 179         char name[RTE_SWX_NAME_SIZE];
 180         struct struct_type *st;
 181         uint32_t struct_id;
 182         uint32_t id;
 183 };
 184
 185 TAILQ_HEAD(header_tailq, header);
 186
 187 struct header_runtime {
 188         uint8_t *ptr0;
 189 };
 190
 191 struct header_out_runtime {
 192         uint8_t *ptr0;
 193         uint8_t *ptr;
 194         uint32_t n_bytes;
 195 };
 196
 197 /*
 198  * Instruction.
 199  */
 200
 201 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 202  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 203  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 204  * when transferred to packet meta-data and in NBO when transferred to packet
 205  * headers.
 206  */
 207
 208 /* Notation conventions:
 209  *    -Header field: H = h.header.field (dst/src)
 210  *    -Meta-data field: M = m.field (dst/src)
 211  *    -Extern object mailbox field: E = e.field (dst/src)
 212  *    -Extern function mailbox field: F = f.field (dst/src)
 213  *    -Table action data field: T = t.field (src only)
 214  *    -Immediate value: I = 32-bit unsigned value (src only)
 215  */
 216
 217 enum instruction_type {
 218         /* rx m.port_in */
 219         INSTR_RX,
 220
 221         /* tx m.port_out */
 222         INSTR_TX,
 223
 224         /* extract h.header */
 225         INSTR_HDR_EXTRACT,
 226         INSTR_HDR_EXTRACT2,
 227         INSTR_HDR_EXTRACT3,
 228         INSTR_HDR_EXTRACT4,
 229         INSTR_HDR_EXTRACT5,
 230         INSTR_HDR_EXTRACT6,
 231         INSTR_HDR_EXTRACT7,
 232         INSTR_HDR_EXTRACT8,
 233
 234         /* emit h.header */
 235         INSTR_HDR_EMIT,
 236         INSTR_HDR_EMIT_TX,
 237         INSTR_HDR_EMIT2_TX,
 238         INSTR_HDR_EMIT3_TX,
 239         INSTR_HDR_EMIT4_TX,
 240         INSTR_HDR_EMIT5_TX,
 241         INSTR_HDR_EMIT6_TX,
 242         INSTR_HDR_EMIT7_TX,
 243         INSTR_HDR_EMIT8_TX,
 244
 245         /* validate h.header */
 246         INSTR_HDR_VALIDATE,
 247
 248         /* invalidate h.header */
 249         INSTR_HDR_INVALIDATE,
 250
 251         /* mov dst src
 252          * dst = src
 253          * dst = HMEF, src = HMEFTI
 254          */
 255         INSTR_MOV,   /* dst = MEF, src = MEFT */
 256         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 257         INSTR_MOV_I, /* dst = HMEF, src = I */
 258
 259         /* dma h.header t.field
 260          * memcpy(h.header, t.field, sizeof(h.header))
 261          */
 262         INSTR_DMA_HT,
 263         INSTR_DMA_HT2,
 264         INSTR_DMA_HT3,
 265         INSTR_DMA_HT4,
 266         INSTR_DMA_HT5,
 267         INSTR_DMA_HT6,
 268         INSTR_DMA_HT7,
 269         INSTR_DMA_HT8,
 270 };
 271
 272 struct instr_operand {
 273         uint8_t struct_id;
 274         uint8_t n_bits;
 275         uint8_t offset;
 276         uint8_t pad;
 277 };
 278
 279 struct instr_io {
 280         struct {
 281                 uint8_t offset;
 282                 uint8_t n_bits;
 283                 uint8_t pad[2];
 284         } io;
 285
 286         struct {
 287                 uint8_t header_id[8];
 288                 uint8_t struct_id[8];
 289                 uint8_t n_bytes[8];
 290         } hdr;
 291 };
 292
 293 struct instr_hdr_validity {
 294         uint8_t header_id;
 295 };
 296
 297 struct instr_dst_src {
 298         struct instr_operand dst;
 299         union {
 300                 struct instr_operand src;
 301                 uint32_t src_val;
 302         };
 303 };
 304
 305 struct instr_dma {
 306         struct {
 307                 uint8_t header_id[8];
 308                 uint8_t struct_id[8];
 309         } dst;
 310
 311         struct {
 312                 uint8_t offset[8];
 313         } src;
 314
 315         uint16_t n_bytes[8];
 316 };
 317
 318 struct instruction {
 319         enum instruction_type type;
 320         union {
 321                 struct instr_io io;
 322                 struct instr_hdr_validity valid;
 323                 struct instr_dst_src mov;
 324                 struct instr_dma dma;
 325         };
 326 };
 327
 328 struct instruction_data {
 329         char label[RTE_SWX_NAME_SIZE];
 330         char jmp_label[RTE_SWX_NAME_SIZE];
 331         uint32_t n_users; /* user = jmp instruction to this instruction. */
 332         int invalid;
 333 };
 334
 335 /*
 336  * Action.
 337  */
 338 struct action {
 339         TAILQ_ENTRY(action) node;
 340         char name[RTE_SWX_NAME_SIZE];
 341         struct struct_type *st;
 342         struct instruction *instructions;
 343         uint32_t n_instructions;
 344         uint32_t id;
 345 };
 346
 347 TAILQ_HEAD(action_tailq, action);
 348
 349 /*
 350  * Table.
 351  */
 352 struct table_type {
 353         TAILQ_ENTRY(table_type) node;
 354         char name[RTE_SWX_NAME_SIZE];
 355         enum rte_swx_table_match_type match_type;
 356         struct rte_swx_table_ops ops;
 357 };
 358
 359 TAILQ_HEAD(table_type_tailq, table_type);
 360
 361 struct match_field {
 362         enum rte_swx_table_match_type match_type;
 363         struct field *field;
 364 };
 365
 366 struct table {
 367         TAILQ_ENTRY(table) node;
 368         char name[RTE_SWX_NAME_SIZE];
 369         char args[RTE_SWX_NAME_SIZE];
 370         struct table_type *type; /* NULL when n_fields == 0. */
 371
 372         /* Match. */
 373         struct match_field *fields;
 374         uint32_t n_fields;
 375         int is_header; /* Only valid when n_fields > 0. */
 376         struct header *header; /* Only valid when n_fields > 0. */
 377
 378         /* Action. */
 379         struct action **actions;
 380         struct action *default_action;
 381         uint8_t *default_action_data;
 382         uint32_t n_actions;
 383         int default_action_is_const;
 384         uint32_t action_data_size_max;
 385
 386         uint32_t size;
 387         uint32_t id;
 388 };
 389
 390 TAILQ_HEAD(table_tailq, table);
 391
 392 struct table_runtime {
 393         rte_swx_table_lookup_t func;
 394         void *mailbox;
 395         uint8_t **key;
 396 };
 397
 398 /*
 399  * Pipeline.
 400  */
 401 struct thread {
 402         /* Packet. */
 403         struct rte_swx_pkt pkt;
 404         uint8_t *ptr;
 405
 406         /* Structures. */
 407         uint8_t **structs;
 408
 409         /* Packet headers. */
 410         struct header_runtime *headers; /* Extracted or generated headers. */
 411         struct header_out_runtime *headers_out; /* Emitted headers. */
 412         uint8_t *header_storage;
 413         uint8_t *header_out_storage;
 414         uint64_t valid_headers;
 415         uint32_t n_headers_out;
 416
 417         /* Packet meta-data. */
 418         uint8_t *metadata;
 419
 420         /* Tables. */
 421         struct table_runtime *tables;
 422         struct rte_swx_table_state *table_state;
 423         uint64_t action_id;
 424         int hit; /* 0 = Miss, 1 = Hit. */
 425
 426         /* Extern objects and functions. */
 427         struct extern_obj_runtime *extern_objs;
 428         struct extern_func_runtime *extern_funcs;
 429
 430         /* Instructions. */
 431         struct instruction *ip;
 432         struct instruction *ret;
 433 };
 434
 435 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 436 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 437 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 438
 439 #define MOV(thread, ip)  \
 440 {                                                                              \
 441         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 442         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 443         uint64_t dst64 = *dst64_ptr;                                           \
 444         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 445                                                                                \
 446         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 447         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 448         uint64_t src64 = *src64_ptr;                                           \
 449         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
 450         uint64_t src = src64 & src64_mask;                                     \
 451                                                                                \
 452         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 453 }
 454
 455 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 456
 457 #define MOV_S(thread, ip)  \
 458 {                                                                              \
 459         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 460         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 461         uint64_t dst64 = *dst64_ptr;                                           \
 462         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 463                                                                                \
 464         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 465         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 466         uint64_t src64 = *src64_ptr;                                           \
 467         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
 468                                                                                \
 469         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 470 }
 471
 472 #else
 473
 474 #define MOV_S MOV
 475
 476 #endif
 477
 478 #define MOV_I(thread, ip)  \
 479 {                                                                              \
 480         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 481         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 482         uint64_t dst64 = *dst64_ptr;                                           \
 483         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 484                                                                                \
 485         uint64_t src = (ip)->mov.src_val;                                      \
 486                                                                                \
 487         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 488 }
 489
 490 #define METADATA_READ(thread, offset, n_bits)                                  \
 491 ({                                                                             \
 492         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 493         uint64_t m64 = *m64_ptr;                                               \
 494         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 495         (m64 & m64_mask);                                                      \
 496 })
 497
 498 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
 499 {                                                                              \
 500         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 501         uint64_t m64 = *m64_ptr;                                               \
 502         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 503                                                                                \
 504         uint64_t m_new = value;                                                \
 505                                                                                \
 506         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
 507 }
 508
 509 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 510 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 511 #endif
 512
 513 struct rte_swx_pipeline {
 514         struct struct_type_tailq struct_types;
 515         struct port_in_type_tailq port_in_types;
 516         struct port_in_tailq ports_in;
 517         struct port_out_type_tailq port_out_types;
 518         struct port_out_tailq ports_out;
 519         struct extern_type_tailq extern_types;
 520         struct extern_obj_tailq extern_objs;
 521         struct extern_func_tailq extern_funcs;
 522         struct header_tailq headers;
 523         struct struct_type *metadata_st;
 524         uint32_t metadata_struct_id;
 525         struct action_tailq actions;
 526         struct table_type_tailq table_types;
 527         struct table_tailq tables;
 528
 529         struct port_in_runtime *in;
 530         struct port_out_runtime *out;
 531         struct instruction **action_instructions;
 532         struct rte_swx_table_state *table_state;
 533         struct instruction *instructions;
 534         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 535
 536         uint32_t n_structs;
 537         uint32_t n_ports_in;
 538         uint32_t n_ports_out;
 539         uint32_t n_extern_objs;
 540         uint32_t n_extern_funcs;
 541         uint32_t n_actions;
 542         uint32_t n_tables;
 543         uint32_t n_headers;
 544         uint32_t thread_id;
 545         uint32_t port_id;
 546         uint32_t n_instructions;
 547         int build_done;
 548         int numa_node;
 549 };
 550
 551 /*
 552  * Struct.
 553  */
 554 static struct struct_type *
 555 struct_type_find(struct rte_swx_pipeline *p, const char *name)
 556 {
 557         struct struct_type *elem;
 558
 559         TAILQ_FOREACH(elem, &p->struct_types, node)
 560                 if (strcmp(elem->name, name) == 0)
 561                         return elem;
 562
 563         return NULL;
 564 }
 565
 566 static struct field *
 567 struct_type_field_find(struct struct_type *st, const char *name)
 568 {
 569         uint32_t i;
 570
 571         for (i = 0; i < st->n_fields; i++) {
 572                 struct field *f = &st->fields[i];
 573
 574                 if (strcmp(f->name, name) == 0)
 575                         return f;
 576         }
 577
 578         return NULL;
 579 }
 580
 581 int
 582 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 583                                       const char *name,
 584                                       struct rte_swx_field_params *fields,
 585                                       uint32_t n_fields)
 586 {
 587         struct struct_type *st;
 588         uint32_t i;
 589
 590         CHECK(p, EINVAL);
 591         CHECK_NAME(name, EINVAL);
 592         CHECK(fields, EINVAL);
 593         CHECK(n_fields, EINVAL);
 594
 595         for (i = 0; i < n_fields; i++) {
 596                 struct rte_swx_field_params *f = &fields[i];
 597                 uint32_t j;
 598
 599                 CHECK_NAME(f->name, EINVAL);
 600                 CHECK(f->n_bits, EINVAL);
 601                 CHECK(f->n_bits <= 64, EINVAL);
 602                 CHECK((f->n_bits & 7) == 0, EINVAL);
 603
 604                 for (j = 0; j < i; j++) {
 605                         struct rte_swx_field_params *f_prev = &fields[j];
 606
 607                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 608                 }
 609         }
 610
 611         CHECK(!struct_type_find(p, name), EEXIST);
 612
 613         /* Node allocation. */
 614         st = calloc(1, sizeof(struct struct_type));
 615         CHECK(st, ENOMEM);
 616
 617         st->fields = calloc(n_fields, sizeof(struct field));
 618         if (!st->fields) {
 619                 free(st);
 620                 CHECK(0, ENOMEM);
 621         }
 622
 623         /* Node initialization. */
 624         strcpy(st->name, name);
 625         for (i = 0; i < n_fields; i++) {
 626                 struct field *dst = &st->fields[i];
 627                 struct rte_swx_field_params *src = &fields[i];
 628
 629                 strcpy(dst->name, src->name);
 630                 dst->n_bits = src->n_bits;
 631                 dst->offset = st->n_bits;
 632
 633                 st->n_bits += src->n_bits;
 634         }
 635         st->n_fields = n_fields;
 636
 637         /* Node add to tailq. */
 638         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 639
 640         return 0;
 641 }
 642
 643 static int
 644 struct_build(struct rte_swx_pipeline *p)
 645 {
 646         uint32_t i;
 647
 648         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 649                 struct thread *t = &p->threads[i];
 650
 651                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 652                 CHECK(t->structs, ENOMEM);
 653         }
 654
 655         return 0;
 656 }
 657
 658 static void
 659 struct_build_free(struct rte_swx_pipeline *p)
 660 {
 661         uint32_t i;
 662
 663         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 664                 struct thread *t = &p->threads[i];
 665
 666                 free(t->structs);
 667                 t->structs = NULL;
 668         }
 669 }
 670
 671 static void
 672 struct_free(struct rte_swx_pipeline *p)
 673 {
 674         struct_build_free(p);
 675
 676         /* Struct types. */
 677         for ( ; ; ) {
 678                 struct struct_type *elem;
 679
 680                 elem = TAILQ_FIRST(&p->struct_types);
 681                 if (!elem)
 682                         break;
 683
 684                 TAILQ_REMOVE(&p->struct_types, elem, node);
 685                 free(elem->fields);
 686                 free(elem);
 687         }
 688 }
 689
 690 /*
 691  * Input port.
 692  */
 693 static struct port_in_type *
 694 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 695 {
 696         struct port_in_type *elem;
 697
 698         if (!name)
 699                 return NULL;
 700
 701         TAILQ_FOREACH(elem, &p->port_in_types, node)
 702                 if (strcmp(elem->name, name) == 0)
 703                         return elem;
 704
 705         return NULL;
 706 }
 707
 708 int
 709 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 710                                        const char *name,
 711                                        struct rte_swx_port_in_ops *ops)
 712 {
 713         struct port_in_type *elem;
 714
 715         CHECK(p, EINVAL);
 716         CHECK_NAME(name, EINVAL);
 717         CHECK(ops, EINVAL);
 718         CHECK(ops->create, EINVAL);
 719         CHECK(ops->free, EINVAL);
 720         CHECK(ops->pkt_rx, EINVAL);
 721         CHECK(ops->stats_read, EINVAL);
 722
 723         CHECK(!port_in_type_find(p, name), EEXIST);
 724
 725         /* Node allocation. */
 726         elem = calloc(1, sizeof(struct port_in_type));
 727         CHECK(elem, ENOMEM);
 728
 729         /* Node initialization. */
 730         strcpy(elem->name, name);
 731         memcpy(&elem->ops, ops, sizeof(*ops));
 732
 733         /* Node add to tailq. */
 734         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 735
 736         return 0;
 737 }
 738
 739 static struct port_in *
 740 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 741 {
 742         struct port_in *port;
 743
 744         TAILQ_FOREACH(port, &p->ports_in, node)
 745                 if (port->id == port_id)
 746                         return port;
 747
 748         return NULL;
 749 }
 750
 751 int
 752 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 753                                 uint32_t port_id,
 754                                 const char *port_type_name,
 755                                 void *args)
 756 {
 757         struct port_in_type *type = NULL;
 758         struct port_in *port = NULL;
 759         void *obj = NULL;
 760
 761         CHECK(p, EINVAL);
 762
 763         CHECK(!port_in_find(p, port_id), EINVAL);
 764
 765         CHECK_NAME(port_type_name, EINVAL);
 766         type = port_in_type_find(p, port_type_name);
 767         CHECK(type, EINVAL);
 768
 769         obj = type->ops.create(args);
 770         CHECK(obj, ENODEV);
 771
 772         /* Node allocation. */
 773         port = calloc(1, sizeof(struct port_in));
 774         CHECK(port, ENOMEM);
 775
 776         /* Node initialization. */
 777         port->type = type;
 778         port->obj = obj;
 779         port->id = port_id;
 780
 781         /* Node add to tailq. */
 782         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 783         if (p->n_ports_in < port_id + 1)
 784                 p->n_ports_in = port_id + 1;
 785
 786         return 0;
 787 }
 788
 789 static int
 790 port_in_build(struct rte_swx_pipeline *p)
 791 {
 792         struct port_in *port;
 793         uint32_t i;
 794
 795         CHECK(p->n_ports_in, EINVAL);
 796         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 797
 798         for (i = 0; i < p->n_ports_in; i++)
 799                 CHECK(port_in_find(p, i), EINVAL);
 800
 801         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 802         CHECK(p->in, ENOMEM);
 803
 804         TAILQ_FOREACH(port, &p->ports_in, node) {
 805                 struct port_in_runtime *in = &p->in[port->id];
 806
 807                 in->pkt_rx = port->type->ops.pkt_rx;
 808                 in->obj = port->obj;
 809         }
 810
 811         return 0;
 812 }
 813
 814 static void
 815 port_in_build_free(struct rte_swx_pipeline *p)
 816 {
 817         free(p->in);
 818         p->in = NULL;
 819 }
 820
 821 static void
 822 port_in_free(struct rte_swx_pipeline *p)
 823 {
 824         port_in_build_free(p);
 825
 826         /* Input ports. */
 827         for ( ; ; ) {
 828                 struct port_in *port;
 829
 830                 port = TAILQ_FIRST(&p->ports_in);
 831                 if (!port)
 832                         break;
 833
 834                 TAILQ_REMOVE(&p->ports_in, port, node);
 835                 port->type->ops.free(port->obj);
 836                 free(port);
 837         }
 838
 839         /* Input port types. */
 840         for ( ; ; ) {
 841                 struct port_in_type *elem;
 842
 843                 elem = TAILQ_FIRST(&p->port_in_types);
 844                 if (!elem)
 845                         break;
 846
 847                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 848                 free(elem);
 849         }
 850 }
 851
 852 /*
 853  * Output port.
 854  */
 855 static struct port_out_type *
 856 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 857 {
 858         struct port_out_type *elem;
 859
 860         if (!name)
 861                 return NULL;
 862
 863         TAILQ_FOREACH(elem, &p->port_out_types, node)
 864                 if (!strcmp(elem->name, name))
 865                         return elem;
 866
 867         return NULL;
 868 }
 869
 870 int
 871 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 872                                         const char *name,
 873                                         struct rte_swx_port_out_ops *ops)
 874 {
 875         struct port_out_type *elem;
 876
 877         CHECK(p, EINVAL);
 878         CHECK_NAME(name, EINVAL);
 879         CHECK(ops, EINVAL);
 880         CHECK(ops->create, EINVAL);
 881         CHECK(ops->free, EINVAL);
 882         CHECK(ops->pkt_tx, EINVAL);
 883         CHECK(ops->stats_read, EINVAL);
 884
 885         CHECK(!port_out_type_find(p, name), EEXIST);
 886
 887         /* Node allocation. */
 888         elem = calloc(1, sizeof(struct port_out_type));
 889         CHECK(elem, ENOMEM);
 890
 891         /* Node initialization. */
 892         strcpy(elem->name, name);
 893         memcpy(&elem->ops, ops, sizeof(*ops));
 894
 895         /* Node add to tailq. */
 896         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 897
 898         return 0;
 899 }
 900
 901 static struct port_out *
 902 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 903 {
 904         struct port_out *port;
 905
 906         TAILQ_FOREACH(port, &p->ports_out, node)
 907                 if (port->id == port_id)
 908                         return port;
 909
 910         return NULL;
 911 }
 912
 913 int
 914 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 915                                  uint32_t port_id,
 916                                  const char *port_type_name,
 917                                  void *args)
 918 {
 919         struct port_out_type *type = NULL;
 920         struct port_out *port = NULL;
 921         void *obj = NULL;
 922
 923         CHECK(p, EINVAL);
 924
 925         CHECK(!port_out_find(p, port_id), EINVAL);
 926
 927         CHECK_NAME(port_type_name, EINVAL);
 928         type = port_out_type_find(p, port_type_name);
 929         CHECK(type, EINVAL);
 930
 931         obj = type->ops.create(args);
 932         CHECK(obj, ENODEV);
 933
 934         /* Node allocation. */
 935         port = calloc(1, sizeof(struct port_out));
 936         CHECK(port, ENOMEM);
 937
 938         /* Node initialization. */
 939         port->type = type;
 940         port->obj = obj;
 941         port->id = port_id;
 942
 943         /* Node add to tailq. */
 944         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 945         if (p->n_ports_out < port_id + 1)
 946                 p->n_ports_out = port_id + 1;
 947
 948         return 0;
 949 }
 950
 951 static int
 952 port_out_build(struct rte_swx_pipeline *p)
 953 {
 954         struct port_out *port;
 955         uint32_t i;
 956
 957         CHECK(p->n_ports_out, EINVAL);
 958
 959         for (i = 0; i < p->n_ports_out; i++)
 960                 CHECK(port_out_find(p, i), EINVAL);
 961
 962         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 963         CHECK(p->out, ENOMEM);
 964
 965         TAILQ_FOREACH(port, &p->ports_out, node) {
 966                 struct port_out_runtime *out = &p->out[port->id];
 967
 968                 out->pkt_tx = port->type->ops.pkt_tx;
 969                 out->flush = port->type->ops.flush;
 970                 out->obj = port->obj;
 971         }
 972
 973         return 0;
 974 }
 975
 976 static void
 977 port_out_build_free(struct rte_swx_pipeline *p)
 978 {
 979         free(p->out);
 980         p->out = NULL;
 981 }
 982
 983 static void
 984 port_out_free(struct rte_swx_pipeline *p)
 985 {
 986         port_out_build_free(p);
 987
 988         /* Output ports. */
 989         for ( ; ; ) {
 990                 struct port_out *port;
 991
 992                 port = TAILQ_FIRST(&p->ports_out);
 993                 if (!port)
 994                         break;
 995
 996                 TAILQ_REMOVE(&p->ports_out, port, node);
 997                 port->type->ops.free(port->obj);
 998                 free(port);
 999         }
1000
1001         /* Output port types. */
1002         for ( ; ; ) {
1003                 struct port_out_type *elem;
1004
1005                 elem = TAILQ_FIRST(&p->port_out_types);
1006                 if (!elem)
1007                         break;
1008
1009                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1010                 free(elem);
1011         }
1012 }
1013
1014 /*
1015  * Extern object.
1016  */
1017 static struct extern_type *
1018 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1019 {
1020         struct extern_type *elem;
1021
1022         TAILQ_FOREACH(elem, &p->extern_types, node)
1023                 if (strcmp(elem->name, name) == 0)
1024                         return elem;
1025
1026         return NULL;
1027 }
1028
1029 static struct extern_type_member_func *
1030 extern_type_member_func_find(struct extern_type *type, const char *name)
1031 {
1032         struct extern_type_member_func *elem;
1033
1034         TAILQ_FOREACH(elem, &type->funcs, node)
1035                 if (strcmp(elem->name, name) == 0)
1036                         return elem;
1037
1038         return NULL;
1039 }
1040
1041 static struct extern_obj *
1042 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1043 {
1044         struct extern_obj *elem;
1045
1046         TAILQ_FOREACH(elem, &p->extern_objs, node)
1047                 if (strcmp(elem->name, name) == 0)
1048                         return elem;
1049
1050         return NULL;
1051 }
1052
1053 static struct field *
1054 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1055                                const char *name,
1056                                struct extern_obj **object)
1057 {
1058         struct extern_obj *obj;
1059         struct field *f;
1060         char *obj_name, *field_name;
1061
1062         if ((name[0] != 'e') || (name[1] != '.'))
1063                 return NULL;
1064
1065         obj_name = strdup(&name[2]);
1066         if (!obj_name)
1067                 return NULL;
1068
1069         field_name = strchr(obj_name, '.');
1070         if (!field_name) {
1071                 free(obj_name);
1072                 return NULL;
1073         }
1074
1075         *field_name = 0;
1076         field_name++;
1077
1078         obj = extern_obj_find(p, obj_name);
1079         if (!obj) {
1080                 free(obj_name);
1081                 return NULL;
1082         }
1083
1084         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1085         if (!f) {
1086                 free(obj_name);
1087                 return NULL;
1088         }
1089
1090         if (object)
1091                 *object = obj;
1092
1093         free(obj_name);
1094         return f;
1095 }
1096
1097 int
1098 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1099         const char *name,
1100         const char *mailbox_struct_type_name,
1101         rte_swx_extern_type_constructor_t constructor,
1102         rte_swx_extern_type_destructor_t destructor)
1103 {
1104         struct extern_type *elem;
1105         struct struct_type *mailbox_struct_type;
1106
1107         CHECK(p, EINVAL);
1108
1109         CHECK_NAME(name, EINVAL);
1110         CHECK(!extern_type_find(p, name), EEXIST);
1111
1112         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1113         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1114         CHECK(mailbox_struct_type, EINVAL);
1115
1116         CHECK(constructor, EINVAL);
1117         CHECK(destructor, EINVAL);
1118
1119         /* Node allocation. */
1120         elem = calloc(1, sizeof(struct extern_type));
1121         CHECK(elem, ENOMEM);
1122
1123         /* Node initialization. */
1124         strcpy(elem->name, name);
1125         elem->mailbox_struct_type = mailbox_struct_type;
1126         elem->constructor = constructor;
1127         elem->destructor = destructor;
1128         TAILQ_INIT(&elem->funcs);
1129
1130         /* Node add to tailq. */
1131         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1132
1133         return 0;
1134 }
1135
1136 int
1137 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1138         const char *extern_type_name,
1139         const char *name,
1140         rte_swx_extern_type_member_func_t member_func)
1141 {
1142         struct extern_type *type;
1143         struct extern_type_member_func *type_member;
1144
1145         CHECK(p, EINVAL);
1146
1147         CHECK(extern_type_name, EINVAL);
1148         type = extern_type_find(p, extern_type_name);
1149         CHECK(type, EINVAL);
1150         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1151
1152         CHECK(name, EINVAL);
1153         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1154
1155         CHECK(member_func, EINVAL);
1156
1157         /* Node allocation. */
1158         type_member = calloc(1, sizeof(struct extern_type_member_func));
1159         CHECK(type_member, ENOMEM);
1160
1161         /* Node initialization. */
1162         strcpy(type_member->name, name);
1163         type_member->func = member_func;
1164         type_member->id = type->n_funcs;
1165
1166         /* Node add to tailq. */
1167         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1168         type->n_funcs++;
1169
1170         return 0;
1171 }
1172
1173 int
1174 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1175                                       const char *extern_type_name,
1176                                       const char *name,
1177                                       const char *args)
1178 {
1179         struct extern_type *type;
1180         struct extern_obj *obj;
1181         void *obj_handle;
1182
1183         CHECK(p, EINVAL);
1184
1185         CHECK_NAME(extern_type_name, EINVAL);
1186         type = extern_type_find(p, extern_type_name);
1187         CHECK(type, EINVAL);
1188
1189         CHECK_NAME(name, EINVAL);
1190         CHECK(!extern_obj_find(p, name), EEXIST);
1191
1192         /* Node allocation. */
1193         obj = calloc(1, sizeof(struct extern_obj));
1194         CHECK(obj, ENOMEM);
1195
1196         /* Object construction. */
1197         obj_handle = type->constructor(args);
1198         if (!obj_handle) {
1199                 free(obj);
1200                 CHECK(0, ENODEV);
1201         }
1202
1203         /* Node initialization. */
1204         strcpy(obj->name, name);
1205         obj->type = type;
1206         obj->obj = obj_handle;
1207         obj->struct_id = p->n_structs;
1208         obj->id = p->n_extern_objs;
1209
1210         /* Node add to tailq. */
1211         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1212         p->n_extern_objs++;
1213         p->n_structs++;
1214
1215         return 0;
1216 }
1217
1218 static int
1219 extern_obj_build(struct rte_swx_pipeline *p)
1220 {
1221         uint32_t i;
1222
1223         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1224                 struct thread *t = &p->threads[i];
1225                 struct extern_obj *obj;
1226
1227                 t->extern_objs = calloc(p->n_extern_objs,
1228                                         sizeof(struct extern_obj_runtime));
1229                 CHECK(t->extern_objs, ENOMEM);
1230
1231                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1232                         struct extern_obj_runtime *r =
1233                                 &t->extern_objs[obj->id];
1234                         struct extern_type_member_func *func;
1235                         uint32_t mailbox_size =
1236                                 obj->type->mailbox_struct_type->n_bits / 8;
1237
1238                         r->obj = obj->obj;
1239
1240                         r->mailbox = calloc(1, mailbox_size);
1241                         CHECK(r->mailbox, ENOMEM);
1242
1243                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1244                                 r->funcs[func->id] = func->func;
1245
1246                         t->structs[obj->struct_id] = r->mailbox;
1247                 }
1248         }
1249
1250         return 0;
1251 }
1252
1253 static void
1254 extern_obj_build_free(struct rte_swx_pipeline *p)
1255 {
1256         uint32_t i;
1257
1258         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1259                 struct thread *t = &p->threads[i];
1260                 uint32_t j;
1261
1262                 if (!t->extern_objs)
1263                         continue;
1264
1265                 for (j = 0; j < p->n_extern_objs; j++) {
1266                         struct extern_obj_runtime *r = &t->extern_objs[j];
1267
1268                         free(r->mailbox);
1269                 }
1270
1271                 free(t->extern_objs);
1272                 t->extern_objs = NULL;
1273         }
1274 }
1275
1276 static void
1277 extern_obj_free(struct rte_swx_pipeline *p)
1278 {
1279         extern_obj_build_free(p);
1280
1281         /* Extern objects. */
1282         for ( ; ; ) {
1283                 struct extern_obj *elem;
1284
1285                 elem = TAILQ_FIRST(&p->extern_objs);
1286                 if (!elem)
1287                         break;
1288
1289                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1290                 if (elem->obj)
1291                         elem->type->destructor(elem->obj);
1292                 free(elem);
1293         }
1294
1295         /* Extern types. */
1296         for ( ; ; ) {
1297                 struct extern_type *elem;
1298
1299                 elem = TAILQ_FIRST(&p->extern_types);
1300                 if (!elem)
1301                         break;
1302
1303                 TAILQ_REMOVE(&p->extern_types, elem, node);
1304
1305                 for ( ; ; ) {
1306                         struct extern_type_member_func *func;
1307
1308                         func = TAILQ_FIRST(&elem->funcs);
1309                         if (!func)
1310                                 break;
1311
1312                         TAILQ_REMOVE(&elem->funcs, func, node);
1313                         free(func);
1314                 }
1315
1316                 free(elem);
1317         }
1318 }
1319
1320 /*
1321  * Extern function.
1322  */
1323 static struct extern_func *
1324 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1325 {
1326         struct extern_func *elem;
1327
1328         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1329                 if (strcmp(elem->name, name) == 0)
1330                         return elem;
1331
1332         return NULL;
1333 }
1334
1335 static struct field *
1336 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1337                                 const char *name,
1338                                 struct extern_func **function)
1339 {
1340         struct extern_func *func;
1341         struct field *f;
1342         char *func_name, *field_name;
1343
1344         if ((name[0] != 'f') || (name[1] != '.'))
1345                 return NULL;
1346
1347         func_name = strdup(&name[2]);
1348         if (!func_name)
1349                 return NULL;
1350
1351         field_name = strchr(func_name, '.');
1352         if (!field_name) {
1353                 free(func_name);
1354                 return NULL;
1355         }
1356
1357         *field_name = 0;
1358         field_name++;
1359
1360         func = extern_func_find(p, func_name);
1361         if (!func) {
1362                 free(func_name);
1363                 return NULL;
1364         }
1365
1366         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1367         if (!f) {
1368                 free(func_name);
1369                 return NULL;
1370         }
1371
1372         if (function)
1373                 *function = func;
1374
1375         free(func_name);
1376         return f;
1377 }
1378
1379 int
1380 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1381                                       const char *name,
1382                                       const char *mailbox_struct_type_name,
1383                                       rte_swx_extern_func_t func)
1384 {
1385         struct extern_func *f;
1386         struct struct_type *mailbox_struct_type;
1387
1388         CHECK(p, EINVAL);
1389
1390         CHECK_NAME(name, EINVAL);
1391         CHECK(!extern_func_find(p, name), EEXIST);
1392
1393         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1394         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1395         CHECK(mailbox_struct_type, EINVAL);
1396
1397         CHECK(func, EINVAL);
1398
1399         /* Node allocation. */
1400         f = calloc(1, sizeof(struct extern_func));
1401         CHECK(func, ENOMEM);
1402
1403         /* Node initialization. */
1404         strcpy(f->name, name);
1405         f->mailbox_struct_type = mailbox_struct_type;
1406         f->func = func;
1407         f->struct_id = p->n_structs;
1408         f->id = p->n_extern_funcs;
1409
1410         /* Node add to tailq. */
1411         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1412         p->n_extern_funcs++;
1413         p->n_structs++;
1414
1415         return 0;
1416 }
1417
1418 static int
1419 extern_func_build(struct rte_swx_pipeline *p)
1420 {
1421         uint32_t i;
1422
1423         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1424                 struct thread *t = &p->threads[i];
1425                 struct extern_func *func;
1426
1427                 /* Memory allocation. */
1428                 t->extern_funcs = calloc(p->n_extern_funcs,
1429                                          sizeof(struct extern_func_runtime));
1430                 CHECK(t->extern_funcs, ENOMEM);
1431
1432                 /* Extern function. */
1433                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1434                         struct extern_func_runtime *r =
1435                                 &t->extern_funcs[func->id];
1436                         uint32_t mailbox_size =
1437                                 func->mailbox_struct_type->n_bits / 8;
1438
1439                         r->func = func->func;
1440
1441                         r->mailbox = calloc(1, mailbox_size);
1442                         CHECK(r->mailbox, ENOMEM);
1443
1444                         t->structs[func->struct_id] = r->mailbox;
1445                 }
1446         }
1447
1448         return 0;
1449 }
1450
1451 static void
1452 extern_func_build_free(struct rte_swx_pipeline *p)
1453 {
1454         uint32_t i;
1455
1456         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1457                 struct thread *t = &p->threads[i];
1458                 uint32_t j;
1459
1460                 if (!t->extern_funcs)
1461                         continue;
1462
1463                 for (j = 0; j < p->n_extern_funcs; j++) {
1464                         struct extern_func_runtime *r = &t->extern_funcs[j];
1465
1466                         free(r->mailbox);
1467                 }
1468
1469                 free(t->extern_funcs);
1470                 t->extern_funcs = NULL;
1471         }
1472 }
1473
1474 static void
1475 extern_func_free(struct rte_swx_pipeline *p)
1476 {
1477         extern_func_build_free(p);
1478
1479         for ( ; ; ) {
1480                 struct extern_func *elem;
1481
1482                 elem = TAILQ_FIRST(&p->extern_funcs);
1483                 if (!elem)
1484                         break;
1485
1486                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1487                 free(elem);
1488         }
1489 }
1490
1491 /*
1492  * Header.
1493  */
1494 static struct header *
1495 header_find(struct rte_swx_pipeline *p, const char *name)
1496 {
1497         struct header *elem;
1498
1499         TAILQ_FOREACH(elem, &p->headers, node)
1500                 if (strcmp(elem->name, name) == 0)
1501                         return elem;
1502
1503         return NULL;
1504 }
1505
1506 static struct header *
1507 header_parse(struct rte_swx_pipeline *p,
1508              const char *name)
1509 {
1510         if (name[0] != 'h' || name[1] != '.')
1511                 return NULL;
1512
1513         return header_find(p, &name[2]);
1514 }
1515
1516 static struct field *
1517 header_field_parse(struct rte_swx_pipeline *p,
1518                    const char *name,
1519                    struct header **header)
1520 {
1521         struct header *h;
1522         struct field *f;
1523         char *header_name, *field_name;
1524
1525         if ((name[0] != 'h') || (name[1] != '.'))
1526                 return NULL;
1527
1528         header_name = strdup(&name[2]);
1529         if (!header_name)
1530                 return NULL;
1531
1532         field_name = strchr(header_name, '.');
1533         if (!field_name) {
1534                 free(header_name);
1535                 return NULL;
1536         }
1537
1538         *field_name = 0;
1539         field_name++;
1540
1541         h = header_find(p, header_name);
1542         if (!h) {
1543                 free(header_name);
1544                 return NULL;
1545         }
1546
1547         f = struct_type_field_find(h->st, field_name);
1548         if (!f) {
1549                 free(header_name);
1550                 return NULL;
1551         }
1552
1553         if (header)
1554                 *header = h;
1555
1556         free(header_name);
1557         return f;
1558 }
1559
1560 int
1561 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1562                                         const char *name,
1563                                         const char *struct_type_name)
1564 {
1565         struct struct_type *st;
1566         struct header *h;
1567         size_t n_headers_max;
1568
1569         CHECK(p, EINVAL);
1570         CHECK_NAME(name, EINVAL);
1571         CHECK_NAME(struct_type_name, EINVAL);
1572
1573         CHECK(!header_find(p, name), EEXIST);
1574
1575         st = struct_type_find(p, struct_type_name);
1576         CHECK(st, EINVAL);
1577
1578         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1579         CHECK(p->n_headers < n_headers_max, ENOSPC);
1580
1581         /* Node allocation. */
1582         h = calloc(1, sizeof(struct header));
1583         CHECK(h, ENOMEM);
1584
1585         /* Node initialization. */
1586         strcpy(h->name, name);
1587         h->st = st;
1588         h->struct_id = p->n_structs;
1589         h->id = p->n_headers;
1590
1591         /* Node add to tailq. */
1592         TAILQ_INSERT_TAIL(&p->headers, h, node);
1593         p->n_headers++;
1594         p->n_structs++;
1595
1596         return 0;
1597 }
1598
1599 static int
1600 header_build(struct rte_swx_pipeline *p)
1601 {
1602         struct header *h;
1603         uint32_t n_bytes = 0, i;
1604
1605         TAILQ_FOREACH(h, &p->headers, node) {
1606                 n_bytes += h->st->n_bits / 8;
1607         }
1608
1609         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1610                 struct thread *t = &p->threads[i];
1611                 uint32_t offset = 0;
1612
1613                 t->headers = calloc(p->n_headers,
1614                                     sizeof(struct header_runtime));
1615                 CHECK(t->headers, ENOMEM);
1616
1617                 t->headers_out = calloc(p->n_headers,
1618                                         sizeof(struct header_out_runtime));
1619                 CHECK(t->headers_out, ENOMEM);
1620
1621                 t->header_storage = calloc(1, n_bytes);
1622                 CHECK(t->header_storage, ENOMEM);
1623
1624                 t->header_out_storage = calloc(1, n_bytes);
1625                 CHECK(t->header_out_storage, ENOMEM);
1626
1627                 TAILQ_FOREACH(h, &p->headers, node) {
1628                         uint8_t *header_storage;
1629
1630                         header_storage = &t->header_storage[offset];
1631                         offset += h->st->n_bits / 8;
1632
1633                         t->headers[h->id].ptr0 = header_storage;
1634                         t->structs[h->struct_id] = header_storage;
1635                 }
1636         }
1637
1638         return 0;
1639 }
1640
1641 static void
1642 header_build_free(struct rte_swx_pipeline *p)
1643 {
1644         uint32_t i;
1645
1646         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1647                 struct thread *t = &p->threads[i];
1648
1649                 free(t->headers_out);
1650                 t->headers_out = NULL;
1651
1652                 free(t->headers);
1653                 t->headers = NULL;
1654
1655                 free(t->header_out_storage);
1656                 t->header_out_storage = NULL;
1657
1658                 free(t->header_storage);
1659                 t->header_storage = NULL;
1660         }
1661 }
1662
1663 static void
1664 header_free(struct rte_swx_pipeline *p)
1665 {
1666         header_build_free(p);
1667
1668         for ( ; ; ) {
1669                 struct header *elem;
1670
1671                 elem = TAILQ_FIRST(&p->headers);
1672                 if (!elem)
1673                         break;
1674
1675                 TAILQ_REMOVE(&p->headers, elem, node);
1676                 free(elem);
1677         }
1678 }
1679
1680 /*
1681  * Meta-data.
1682  */
1683 static struct field *
1684 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1685 {
1686         if (!p->metadata_st)
1687                 return NULL;
1688
1689         if (name[0] != 'm' || name[1] != '.')
1690                 return NULL;
1691
1692         return struct_type_field_find(p->metadata_st, &name[2]);
1693 }
1694
1695 int
1696 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1697                                           const char *struct_type_name)
1698 {
1699         struct struct_type *st = NULL;
1700
1701         CHECK(p, EINVAL);
1702
1703         CHECK_NAME(struct_type_name, EINVAL);
1704         st  = struct_type_find(p, struct_type_name);
1705         CHECK(st, EINVAL);
1706         CHECK(!p->metadata_st, EINVAL);
1707
1708         p->metadata_st = st;
1709         p->metadata_struct_id = p->n_structs;
1710
1711         p->n_structs++;
1712
1713         return 0;
1714 }
1715
1716 static int
1717 metadata_build(struct rte_swx_pipeline *p)
1718 {
1719         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1720         uint32_t i;
1721
1722         /* Thread-level initialization. */
1723         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1724                 struct thread *t = &p->threads[i];
1725                 uint8_t *metadata;
1726
1727                 metadata = calloc(1, n_bytes);
1728                 CHECK(metadata, ENOMEM);
1729
1730                 t->metadata = metadata;
1731                 t->structs[p->metadata_struct_id] = metadata;
1732         }
1733
1734         return 0;
1735 }
1736
1737 static void
1738 metadata_build_free(struct rte_swx_pipeline *p)
1739 {
1740         uint32_t i;
1741
1742         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1743                 struct thread *t = &p->threads[i];
1744
1745                 free(t->metadata);
1746                 t->metadata = NULL;
1747         }
1748 }
1749
1750 static void
1751 metadata_free(struct rte_swx_pipeline *p)
1752 {
1753         metadata_build_free(p);
1754 }
1755
1756 /*
1757  * Instruction.
1758  */
1759 static struct field *
1760 action_field_parse(struct action *action, const char *name);
1761
1762 static struct field *
1763 struct_field_parse(struct rte_swx_pipeline *p,
1764                    struct action *action,
1765                    const char *name,
1766                    uint32_t *struct_id)
1767 {
1768         struct field *f;
1769
1770         switch (name[0]) {
1771         case 'h':
1772         {
1773                 struct header *header;
1774
1775                 f = header_field_parse(p, name, &header);
1776                 if (!f)
1777                         return NULL;
1778
1779                 *struct_id = header->struct_id;
1780                 return f;
1781         }
1782
1783         case 'm':
1784         {
1785                 f = metadata_field_parse(p, name);
1786                 if (!f)
1787                         return NULL;
1788
1789                 *struct_id = p->metadata_struct_id;
1790                 return f;
1791         }
1792
1793         case 't':
1794         {
1795                 if (!action)
1796                         return NULL;
1797
1798                 f = action_field_parse(action, name);
1799                 if (!f)
1800                         return NULL;
1801
1802                 *struct_id = 0;
1803                 return f;
1804         }
1805
1806         case 'e':
1807         {
1808                 struct extern_obj *obj;
1809
1810                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1811                 if (!f)
1812                         return NULL;
1813
1814                 *struct_id = obj->struct_id;
1815                 return f;
1816         }
1817
1818         case 'f':
1819         {
1820                 struct extern_func *func;
1821
1822                 f = extern_func_mailbox_field_parse(p, name, &func);
1823                 if (!f)
1824                         return NULL;
1825
1826                 *struct_id = func->struct_id;
1827                 return f;
1828         }
1829
1830         default:
1831                 return NULL;
1832         }
1833 }
1834
1835 static inline void
1836 pipeline_port_inc(struct rte_swx_pipeline *p)
1837 {
1838         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
1839 }
1840
1841 static inline void
1842 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1843 {
1844         t->ip = p->instructions;
1845 }
1846
1847 static inline void
1848 thread_ip_inc(struct rte_swx_pipeline *p);
1849
1850 static inline void
1851 thread_ip_inc(struct rte_swx_pipeline *p)
1852 {
1853         struct thread *t = &p->threads[p->thread_id];
1854
1855         t->ip++;
1856 }
1857
1858 static inline void
1859 thread_ip_inc_cond(struct thread *t, int cond)
1860 {
1861         t->ip += cond;
1862 }
1863
1864 static inline void
1865 thread_yield(struct rte_swx_pipeline *p)
1866 {
1867         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1868 }
1869
1870 /*
1871  * rx.
1872  */
1873 static int
1874 instr_rx_translate(struct rte_swx_pipeline *p,
1875                    struct action *action,
1876                    char **tokens,
1877                    int n_tokens,
1878                    struct instruction *instr,
1879                    struct instruction_data *data __rte_unused)
1880 {
1881         struct field *f;
1882
1883         CHECK(!action, EINVAL);
1884         CHECK(n_tokens == 2, EINVAL);
1885
1886         f = metadata_field_parse(p, tokens[1]);
1887         CHECK(f, EINVAL);
1888
1889         instr->type = INSTR_RX;
1890         instr->io.io.offset = f->offset / 8;
1891         instr->io.io.n_bits = f->n_bits;
1892         return 0;
1893 }
1894
1895 static inline void
1896 instr_rx_exec(struct rte_swx_pipeline *p);
1897
1898 static inline void
1899 instr_rx_exec(struct rte_swx_pipeline *p)
1900 {
1901         struct thread *t = &p->threads[p->thread_id];
1902         struct instruction *ip = t->ip;
1903         struct port_in_runtime *port = &p->in[p->port_id];
1904         struct rte_swx_pkt *pkt = &t->pkt;
1905         int pkt_received;
1906
1907         /* Packet. */
1908         pkt_received = port->pkt_rx(port->obj, pkt);
1909         t->ptr = &pkt->pkt[pkt->offset];
1910         rte_prefetch0(t->ptr);
1911
1912         TRACE("[Thread %2u] rx %s from port %u\n",
1913               p->thread_id,
1914               pkt_received ? "1 pkt" : "0 pkts",
1915               p->port_id);
1916
1917         /* Headers. */
1918         t->valid_headers = 0;
1919         t->n_headers_out = 0;
1920
1921         /* Meta-data. */
1922         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1923
1924         /* Tables. */
1925         t->table_state = p->table_state;
1926
1927         /* Thread. */
1928         pipeline_port_inc(p);
1929         thread_ip_inc_cond(t, pkt_received);
1930         thread_yield(p);
1931 }
1932
1933 /*
1934  * tx.
1935  */
1936 static int
1937 instr_tx_translate(struct rte_swx_pipeline *p,
1938                    struct action *action __rte_unused,
1939                    char **tokens,
1940                    int n_tokens,
1941                    struct instruction *instr,
1942                    struct instruction_data *data __rte_unused)
1943 {
1944         struct field *f;
1945
1946         CHECK(n_tokens == 2, EINVAL);
1947
1948         f = metadata_field_parse(p, tokens[1]);
1949         CHECK(f, EINVAL);
1950
1951         instr->type = INSTR_TX;
1952         instr->io.io.offset = f->offset / 8;
1953         instr->io.io.n_bits = f->n_bits;
1954         return 0;
1955 }
1956
1957 static inline void
1958 emit_handler(struct thread *t)
1959 {
1960         struct header_out_runtime *h0 = &t->headers_out[0];
1961         struct header_out_runtime *h1 = &t->headers_out[1];
1962         uint32_t offset = 0, i;
1963
1964         /* No header change or header decapsulation. */
1965         if ((t->n_headers_out == 1) &&
1966             (h0->ptr + h0->n_bytes == t->ptr)) {
1967                 TRACE("Emit handler: no header change or header decap.\n");
1968
1969                 t->pkt.offset -= h0->n_bytes;
1970                 t->pkt.length += h0->n_bytes;
1971
1972                 return;
1973         }
1974
1975         /* Header encapsulation (optionally, with prior header decasulation). */
1976         if ((t->n_headers_out == 2) &&
1977             (h1->ptr + h1->n_bytes == t->ptr) &&
1978             (h0->ptr == h0->ptr0)) {
1979                 uint32_t offset;
1980
1981                 TRACE("Emit handler: header encapsulation.\n");
1982
1983                 offset = h0->n_bytes + h1->n_bytes;
1984                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1985                 t->pkt.offset -= offset;
1986                 t->pkt.length += offset;
1987
1988                 return;
1989         }
1990
1991         /* Header insertion. */
1992         /* TBD */
1993
1994         /* Header extraction. */
1995         /* TBD */
1996
1997         /* For any other case. */
1998         TRACE("Emit handler: complex case.\n");
1999
2000         for (i = 0; i < t->n_headers_out; i++) {
2001                 struct header_out_runtime *h = &t->headers_out[i];
2002
2003                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2004                 offset += h->n_bytes;
2005         }
2006
2007         if (offset) {
2008                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2009                 t->pkt.offset -= offset;
2010                 t->pkt.length += offset;
2011         }
2012 }
2013
2014 static inline void
2015 instr_tx_exec(struct rte_swx_pipeline *p);
2016
2017 static inline void
2018 instr_tx_exec(struct rte_swx_pipeline *p)
2019 {
2020         struct thread *t = &p->threads[p->thread_id];
2021         struct instruction *ip = t->ip;
2022         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2023         struct port_out_runtime *port = &p->out[port_id];
2024         struct rte_swx_pkt *pkt = &t->pkt;
2025
2026         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2027               p->thread_id,
2028               (uint32_t)port_id);
2029
2030         /* Headers. */
2031         emit_handler(t);
2032
2033         /* Packet. */
2034         port->pkt_tx(port->obj, pkt);
2035
2036         /* Thread. */
2037         thread_ip_reset(p, t);
2038         instr_rx_exec(p);
2039 }
2040
2041 /*
2042  * extract.
2043  */
2044 static int
2045 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2046                             struct action *action,
2047                             char **tokens,
2048                             int n_tokens,
2049                             struct instruction *instr,
2050                             struct instruction_data *data __rte_unused)
2051 {
2052         struct header *h;
2053
2054         CHECK(!action, EINVAL);
2055         CHECK(n_tokens == 2, EINVAL);
2056
2057         h = header_parse(p, tokens[1]);
2058         CHECK(h, EINVAL);
2059
2060         instr->type = INSTR_HDR_EXTRACT;
2061         instr->io.hdr.header_id[0] = h->id;
2062         instr->io.hdr.struct_id[0] = h->struct_id;
2063         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2064         return 0;
2065 }
2066
2067 static inline void
2068 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2069
2070 static inline void
2071 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2072 {
2073         struct thread *t = &p->threads[p->thread_id];
2074         struct instruction *ip = t->ip;
2075         uint64_t valid_headers = t->valid_headers;
2076         uint8_t *ptr = t->ptr;
2077         uint32_t offset = t->pkt.offset;
2078         uint32_t length = t->pkt.length;
2079         uint32_t i;
2080
2081         for (i = 0; i < n_extract; i++) {
2082                 uint32_t header_id = ip->io.hdr.header_id[i];
2083                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2084                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2085
2086                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2087                       p->thread_id,
2088                       header_id,
2089                       n_bytes);
2090
2091                 /* Headers. */
2092                 t->structs[struct_id] = ptr;
2093                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2094
2095                 /* Packet. */
2096                 offset += n_bytes;
2097                 length -= n_bytes;
2098                 ptr += n_bytes;
2099         }
2100
2101         /* Headers. */
2102         t->valid_headers = valid_headers;
2103
2104         /* Packet. */
2105         t->pkt.offset = offset;
2106         t->pkt.length = length;
2107         t->ptr = ptr;
2108 }
2109
2110 static inline void
2111 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2112 {
2113         __instr_hdr_extract_exec(p, 1);
2114
2115         /* Thread. */
2116         thread_ip_inc(p);
2117 }
2118
2119 static inline void
2120 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2121 {
2122         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2123               p->thread_id);
2124
2125         __instr_hdr_extract_exec(p, 2);
2126
2127         /* Thread. */
2128         thread_ip_inc(p);
2129 }
2130
2131 static inline void
2132 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2133 {
2134         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2135               p->thread_id);
2136
2137         __instr_hdr_extract_exec(p, 3);
2138
2139         /* Thread. */
2140         thread_ip_inc(p);
2141 }
2142
2143 static inline void
2144 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2145 {
2146         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2147               p->thread_id);
2148
2149         __instr_hdr_extract_exec(p, 4);
2150
2151         /* Thread. */
2152         thread_ip_inc(p);
2153 }
2154
2155 static inline void
2156 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2157 {
2158         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2159               p->thread_id);
2160
2161         __instr_hdr_extract_exec(p, 5);
2162
2163         /* Thread. */
2164         thread_ip_inc(p);
2165 }
2166
2167 static inline void
2168 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2169 {
2170         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2171               p->thread_id);
2172
2173         __instr_hdr_extract_exec(p, 6);
2174
2175         /* Thread. */
2176         thread_ip_inc(p);
2177 }
2178
2179 static inline void
2180 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2181 {
2182         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2183               p->thread_id);
2184
2185         __instr_hdr_extract_exec(p, 7);
2186
2187         /* Thread. */
2188         thread_ip_inc(p);
2189 }
2190
2191 static inline void
2192 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2193 {
2194         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2195               p->thread_id);
2196
2197         __instr_hdr_extract_exec(p, 8);
2198
2199         /* Thread. */
2200         thread_ip_inc(p);
2201 }
2202
2203 /*
2204  * emit.
2205  */
2206 static int
2207 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2208                          struct action *action __rte_unused,
2209                          char **tokens,
2210                          int n_tokens,
2211                          struct instruction *instr,
2212                          struct instruction_data *data __rte_unused)
2213 {
2214         struct header *h;
2215
2216         CHECK(n_tokens == 2, EINVAL);
2217
2218         h = header_parse(p, tokens[1]);
2219         CHECK(h, EINVAL);
2220
2221         instr->type = INSTR_HDR_EMIT;
2222         instr->io.hdr.header_id[0] = h->id;
2223         instr->io.hdr.struct_id[0] = h->struct_id;
2224         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2225         return 0;
2226 }
2227
2228 static inline void
2229 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
2230
2231 static inline void
2232 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
2233 {
2234         struct thread *t = &p->threads[p->thread_id];
2235         struct instruction *ip = t->ip;
2236         uint32_t n_headers_out = t->n_headers_out;
2237         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
2238         uint8_t *ho_ptr = NULL;
2239         uint32_t ho_nbytes = 0, i;
2240
2241         for (i = 0; i < n_emit; i++) {
2242                 uint32_t header_id = ip->io.hdr.header_id[i];
2243                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2244                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2245
2246                 struct header_runtime *hi = &t->headers[header_id];
2247                 uint8_t *hi_ptr = t->structs[struct_id];
2248
2249                 TRACE("[Thread %2u]: emit header %u\n",
2250                       p->thread_id,
2251                       header_id);
2252
2253                 /* Headers. */
2254                 if (!i) {
2255                         if (!t->n_headers_out) {
2256                                 ho = &t->headers_out[0];
2257
2258                                 ho->ptr0 = hi->ptr0;
2259                                 ho->ptr = hi_ptr;
2260
2261                                 ho_ptr = hi_ptr;
2262                                 ho_nbytes = n_bytes;
2263
2264                                 n_headers_out = 1;
2265
2266                                 continue;
2267                         } else {
2268                                 ho_ptr = ho->ptr;
2269                                 ho_nbytes = ho->n_bytes;
2270                         }
2271                 }
2272
2273                 if (ho_ptr + ho_nbytes == hi_ptr) {
2274                         ho_nbytes += n_bytes;
2275                 } else {
2276                         ho->n_bytes = ho_nbytes;
2277
2278                         ho++;
2279                         ho->ptr0 = hi->ptr0;
2280                         ho->ptr = hi_ptr;
2281
2282                         ho_ptr = hi_ptr;
2283                         ho_nbytes = n_bytes;
2284
2285                         n_headers_out++;
2286                 }
2287         }
2288
2289         ho->n_bytes = ho_nbytes;
2290         t->n_headers_out = n_headers_out;
2291 }
2292
2293 static inline void
2294 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2295 {
2296         __instr_hdr_emit_exec(p, 1);
2297
2298         /* Thread. */
2299         thread_ip_inc(p);
2300 }
2301
2302 static inline void
2303 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2304 {
2305         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2306               p->thread_id);
2307
2308         __instr_hdr_emit_exec(p, 1);
2309         instr_tx_exec(p);
2310 }
2311
2312 static inline void
2313 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2314 {
2315         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2316               p->thread_id);
2317
2318         __instr_hdr_emit_exec(p, 2);
2319         instr_tx_exec(p);
2320 }
2321
2322 static inline void
2323 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2324 {
2325         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2326               p->thread_id);
2327
2328         __instr_hdr_emit_exec(p, 3);
2329         instr_tx_exec(p);
2330 }
2331
2332 static inline void
2333 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2334 {
2335         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2336               p->thread_id);
2337
2338         __instr_hdr_emit_exec(p, 4);
2339         instr_tx_exec(p);
2340 }
2341
2342 static inline void
2343 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2344 {
2345         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2346               p->thread_id);
2347
2348         __instr_hdr_emit_exec(p, 5);
2349         instr_tx_exec(p);
2350 }
2351
2352 static inline void
2353 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2354 {
2355         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2356               p->thread_id);
2357
2358         __instr_hdr_emit_exec(p, 6);
2359         instr_tx_exec(p);
2360 }
2361
2362 static inline void
2363 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2364 {
2365         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2366               p->thread_id);
2367
2368         __instr_hdr_emit_exec(p, 7);
2369         instr_tx_exec(p);
2370 }
2371
2372 static inline void
2373 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2374 {
2375         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2376               p->thread_id);
2377
2378         __instr_hdr_emit_exec(p, 8);
2379         instr_tx_exec(p);
2380 }
2381
2382 /*
2383  * validate.
2384  */
2385 static int
2386 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2387                              struct action *action __rte_unused,
2388                              char **tokens,
2389                              int n_tokens,
2390                              struct instruction *instr,
2391                              struct instruction_data *data __rte_unused)
2392 {
2393         struct header *h;
2394
2395         CHECK(n_tokens == 2, EINVAL);
2396
2397         h = header_parse(p, tokens[1]);
2398         CHECK(h, EINVAL);
2399
2400         instr->type = INSTR_HDR_VALIDATE;
2401         instr->valid.header_id = h->id;
2402         return 0;
2403 }
2404
2405 static inline void
2406 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2407 {
2408         struct thread *t = &p->threads[p->thread_id];
2409         struct instruction *ip = t->ip;
2410         uint32_t header_id = ip->valid.header_id;
2411
2412         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2413
2414         /* Headers. */
2415         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2416
2417         /* Thread. */
2418         thread_ip_inc(p);
2419 }
2420
2421 /*
2422  * invalidate.
2423  */
2424 static int
2425 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2426                                struct action *action __rte_unused,
2427                                char **tokens,
2428                                int n_tokens,
2429                                struct instruction *instr,
2430                                struct instruction_data *data __rte_unused)
2431 {
2432         struct header *h;
2433
2434         CHECK(n_tokens == 2, EINVAL);
2435
2436         h = header_parse(p, tokens[1]);
2437         CHECK(h, EINVAL);
2438
2439         instr->type = INSTR_HDR_INVALIDATE;
2440         instr->valid.header_id = h->id;
2441         return 0;
2442 }
2443
2444 static inline void
2445 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2446 {
2447         struct thread *t = &p->threads[p->thread_id];
2448         struct instruction *ip = t->ip;
2449         uint32_t header_id = ip->valid.header_id;
2450
2451         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2452
2453         /* Headers. */
2454         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2455
2456         /* Thread. */
2457         thread_ip_inc(p);
2458 }
2459
2460 /*
2461  * mov.
2462  */
2463 static int
2464 instr_mov_translate(struct rte_swx_pipeline *p,
2465                     struct action *action,
2466                     char **tokens,
2467                     int n_tokens,
2468                     struct instruction *instr,
2469                     struct instruction_data *data __rte_unused)
2470 {
2471         char *dst = tokens[1], *src = tokens[2];
2472         struct field *fdst, *fsrc;
2473         uint32_t dst_struct_id, src_struct_id, src_val;
2474
2475         CHECK(n_tokens == 3, EINVAL);
2476
2477         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2478         CHECK(fdst, EINVAL);
2479
2480         /* MOV or MOV_S. */
2481         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2482         if (fsrc) {
2483                 instr->type = INSTR_MOV;
2484                 if ((dst[0] == 'h' && src[0] != 'h') ||
2485                     (dst[0] != 'h' && src[0] == 'h'))
2486                         instr->type = INSTR_MOV_S;
2487
2488                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2489                 instr->mov.dst.n_bits = fdst->n_bits;
2490                 instr->mov.dst.offset = fdst->offset / 8;
2491                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2492                 instr->mov.src.n_bits = fsrc->n_bits;
2493                 instr->mov.src.offset = fsrc->offset / 8;
2494                 return 0;
2495         }
2496
2497         /* MOV_I. */
2498         src_val = strtoul(src, &src, 0);
2499         CHECK(!src[0], EINVAL);
2500
2501         if (dst[0] == 'h')
2502                 src_val = htonl(src_val);
2503
2504         instr->type = INSTR_MOV_I;
2505         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2506         instr->mov.dst.n_bits = fdst->n_bits;
2507         instr->mov.dst.offset = fdst->offset / 8;
2508         instr->mov.src_val = (uint32_t)src_val;
2509         return 0;
2510 }
2511
2512 static inline void
2513 instr_mov_exec(struct rte_swx_pipeline *p)
2514 {
2515         struct thread *t = &p->threads[p->thread_id];
2516         struct instruction *ip = t->ip;
2517
2518         TRACE("[Thread %2u] mov\n",
2519               p->thread_id);
2520
2521         MOV(t, ip);
2522
2523         /* Thread. */
2524         thread_ip_inc(p);
2525 }
2526
2527 static inline void
2528 instr_mov_s_exec(struct rte_swx_pipeline *p)
2529 {
2530         struct thread *t = &p->threads[p->thread_id];
2531         struct instruction *ip = t->ip;
2532
2533         TRACE("[Thread %2u] mov (s)\n",
2534               p->thread_id);
2535
2536         MOV_S(t, ip);
2537
2538         /* Thread. */
2539         thread_ip_inc(p);
2540 }
2541
2542 static inline void
2543 instr_mov_i_exec(struct rte_swx_pipeline *p)
2544 {
2545         struct thread *t = &p->threads[p->thread_id];
2546         struct instruction *ip = t->ip;
2547
2548         TRACE("[Thread %2u] mov m.f %x\n",
2549               p->thread_id,
2550               ip->mov.src_val);
2551
2552         MOV_I(t, ip);
2553
2554         /* Thread. */
2555         thread_ip_inc(p);
2556 }
2557
2558 /*
2559  * dma.
2560  */
2561 static int
2562 instr_dma_translate(struct rte_swx_pipeline *p,
2563                     struct action *action,
2564                     char **tokens,
2565                     int n_tokens,
2566                     struct instruction *instr,
2567                     struct instruction_data *data __rte_unused)
2568 {
2569         char *dst = tokens[1];
2570         char *src = tokens[2];
2571         struct header *h;
2572         struct field *tf;
2573
2574         CHECK(action, EINVAL);
2575         CHECK(n_tokens == 3, EINVAL);
2576
2577         h = header_parse(p, dst);
2578         CHECK(h, EINVAL);
2579
2580         tf = action_field_parse(action, src);
2581         CHECK(tf, EINVAL);
2582
2583         instr->type = INSTR_DMA_HT;
2584         instr->dma.dst.header_id[0] = h->id;
2585         instr->dma.dst.struct_id[0] = h->struct_id;
2586         instr->dma.n_bytes[0] = h->st->n_bits / 8;
2587         instr->dma.src.offset[0] = tf->offset / 8;
2588
2589         return 0;
2590 }
2591
2592 static inline void
2593 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2594
2595 static inline void
2596 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2597 {
2598         struct thread *t = &p->threads[p->thread_id];
2599         struct instruction *ip = t->ip;
2600         uint8_t *action_data = t->structs[0];
2601         uint64_t valid_headers = t->valid_headers;
2602         uint32_t i;
2603
2604         for (i = 0; i < n_dma; i++) {
2605                 uint32_t header_id = ip->dma.dst.header_id[i];
2606                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2607                 uint32_t offset = ip->dma.src.offset[i];
2608                 uint32_t n_bytes = ip->dma.n_bytes[i];
2609
2610                 struct header_runtime *h = &t->headers[header_id];
2611                 uint8_t *h_ptr0 = h->ptr0;
2612                 uint8_t *h_ptr = t->structs[struct_id];
2613
2614                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2615                         h_ptr : h_ptr0;
2616                 void *src = &action_data[offset];
2617
2618                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2619
2620                 /* Headers. */
2621                 memcpy(dst, src, n_bytes);
2622                 t->structs[struct_id] = dst;
2623                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2624         }
2625
2626         t->valid_headers = valid_headers;
2627 }
2628
2629 static inline void
2630 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2631 {
2632         __instr_dma_ht_exec(p, 1);
2633
2634         /* Thread. */
2635         thread_ip_inc(p);
2636 }
2637
2638 static inline void
2639 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2640 {
2641         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2642               p->thread_id);
2643
2644         __instr_dma_ht_exec(p, 2);
2645
2646         /* Thread. */
2647         thread_ip_inc(p);
2648 }
2649
2650 static inline void
2651 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2652 {
2653         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2654               p->thread_id);
2655
2656         __instr_dma_ht_exec(p, 3);
2657
2658         /* Thread. */
2659         thread_ip_inc(p);
2660 }
2661
2662 static inline void
2663 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2664 {
2665         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2666               p->thread_id);
2667
2668         __instr_dma_ht_exec(p, 4);
2669
2670         /* Thread. */
2671         thread_ip_inc(p);
2672 }
2673
2674 static inline void
2675 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2676 {
2677         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2678               p->thread_id);
2679
2680         __instr_dma_ht_exec(p, 5);
2681
2682         /* Thread. */
2683         thread_ip_inc(p);
2684 }
2685
2686 static inline void
2687 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2688 {
2689         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2690               p->thread_id);
2691
2692         __instr_dma_ht_exec(p, 6);
2693
2694         /* Thread. */
2695         thread_ip_inc(p);
2696 }
2697
2698 static inline void
2699 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2700 {
2701         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2702               p->thread_id);
2703
2704         __instr_dma_ht_exec(p, 7);
2705
2706         /* Thread. */
2707         thread_ip_inc(p);
2708 }
2709
2710 static inline void
2711 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2712 {
2713         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2714               p->thread_id);
2715
2716         __instr_dma_ht_exec(p, 8);
2717
2718         /* Thread. */
2719         thread_ip_inc(p);
2720 }
2721
2722 #define RTE_SWX_INSTRUCTION_TOKENS_MAX 16
2723
2724 static int
2725 instr_translate(struct rte_swx_pipeline *p,
2726                 struct action *action,
2727                 char *string,
2728                 struct instruction *instr,
2729                 struct instruction_data *data)
2730 {
2731         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
2732         int n_tokens = 0, tpos = 0;
2733
2734         /* Parse the instruction string into tokens. */
2735         for ( ; ; ) {
2736                 char *token;
2737
2738                 token = strtok_r(string, " \t\v", &string);
2739                 if (!token)
2740                         break;
2741
2742                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
2743
2744                 tokens[n_tokens] = token;
2745                 n_tokens++;
2746         }
2747
2748         CHECK(n_tokens, EINVAL);
2749
2750         /* Handle the optional instruction label. */
2751         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
2752                 strcpy(data->label, tokens[0]);
2753
2754                 tpos += 2;
2755                 CHECK(n_tokens - tpos, EINVAL);
2756         }
2757
2758         /* Identify the instruction type. */
2759         if (!strcmp(tokens[tpos], "rx"))
2760                 return instr_rx_translate(p,
2761                                           action,
2762                                           &tokens[tpos],
2763                                           n_tokens - tpos,
2764                                           instr,
2765                                           data);
2766
2767         if (!strcmp(tokens[tpos], "tx"))
2768                 return instr_tx_translate(p,
2769                                           action,
2770                                           &tokens[tpos],
2771                                           n_tokens - tpos,
2772                                           instr,
2773                                           data);
2774
2775         if (!strcmp(tokens[tpos], "extract"))
2776                 return instr_hdr_extract_translate(p,
2777                                                    action,
2778                                                    &tokens[tpos],
2779                                                    n_tokens - tpos,
2780                                                    instr,
2781                                                    data);
2782
2783         if (!strcmp(tokens[tpos], "emit"))
2784                 return instr_hdr_emit_translate(p,
2785                                                 action,
2786                                                 &tokens[tpos],
2787                                                 n_tokens - tpos,
2788                                                 instr,
2789                                                 data);
2790
2791         if (!strcmp(tokens[tpos], "validate"))
2792                 return instr_hdr_validate_translate(p,
2793                                                     action,
2794                                                     &tokens[tpos],
2795                                                     n_tokens - tpos,
2796                                                     instr,
2797                                                     data);
2798
2799         if (!strcmp(tokens[tpos], "invalidate"))
2800                 return instr_hdr_invalidate_translate(p,
2801                                                       action,
2802                                                       &tokens[tpos],
2803                                                       n_tokens - tpos,
2804                                                       instr,
2805                                                       data);
2806
2807         if (!strcmp(tokens[tpos], "mov"))
2808                 return instr_mov_translate(p,
2809                                            action,
2810                                            &tokens[tpos],
2811                                            n_tokens - tpos,
2812                                            instr,
2813                                            data);
2814
2815         if (!strcmp(tokens[tpos], "dma"))
2816                 return instr_dma_translate(p,
2817                                            action,
2818                                            &tokens[tpos],
2819                                            n_tokens - tpos,
2820                                            instr,
2821                                            data);
2822
2823         CHECK(0, EINVAL);
2824 }
2825
2826 static uint32_t
2827 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
2828 {
2829         uint32_t count = 0, i;
2830
2831         if (!label[0])
2832                 return 0;
2833
2834         for (i = 0; i < n; i++)
2835                 if (!strcmp(label, data[i].jmp_label))
2836                         count++;
2837
2838         return count;
2839 }
2840
2841 static int
2842 instr_label_check(struct instruction_data *instruction_data,
2843                   uint32_t n_instructions)
2844 {
2845         uint32_t i;
2846
2847         /* Check that all instruction labels are unique. */
2848         for (i = 0; i < n_instructions; i++) {
2849                 struct instruction_data *data = &instruction_data[i];
2850                 char *label = data->label;
2851                 uint32_t j;
2852
2853                 if (!label[0])
2854                         continue;
2855
2856                 for (j = i + 1; j < n_instructions; j++)
2857                         CHECK(strcmp(label, data[j].label), EINVAL);
2858         }
2859
2860         /* Get users for each instruction label. */
2861         for (i = 0; i < n_instructions; i++) {
2862                 struct instruction_data *data = &instruction_data[i];
2863                 char *label = data->label;
2864
2865                 data->n_users = label_is_used(instruction_data,
2866                                               n_instructions,
2867                                               label);
2868         }
2869
2870         return 0;
2871 }
2872
2873 static int
2874 instruction_config(struct rte_swx_pipeline *p,
2875                    struct action *a,
2876                    const char **instructions,
2877                    uint32_t n_instructions)
2878 {
2879         struct instruction *instr = NULL;
2880         struct instruction_data *data = NULL;
2881         char *string = NULL;
2882         int err = 0;
2883         uint32_t i;
2884
2885         CHECK(n_instructions, EINVAL);
2886         CHECK(instructions, EINVAL);
2887         for (i = 0; i < n_instructions; i++)
2888                 CHECK(instructions[i], EINVAL);
2889
2890         /* Memory allocation. */
2891         instr = calloc(n_instructions, sizeof(struct instruction));
2892         if (!instr) {
2893                 err = ENOMEM;
2894                 goto error;
2895         }
2896
2897         data = calloc(n_instructions, sizeof(struct instruction_data));
2898         if (!data) {
2899                 err = ENOMEM;
2900                 goto error;
2901         }
2902
2903         for (i = 0; i < n_instructions; i++) {
2904                 string = strdup(instructions[i]);
2905                 if (!string) {
2906                         err = ENOMEM;
2907                         goto error;
2908                 }
2909
2910                 err = instr_translate(p, a, string, &instr[i], &data[i]);
2911                 if (err)
2912                         goto error;
2913
2914                 free(string);
2915         }
2916
2917         err = instr_label_check(data, n_instructions);
2918         if (err)
2919                 goto error;
2920
2921         free(data);
2922
2923         if (a) {
2924                 a->instructions = instr;
2925                 a->n_instructions = n_instructions;
2926         } else {
2927                 p->instructions = instr;
2928                 p->n_instructions = n_instructions;
2929         }
2930
2931         return 0;
2932
2933 error:
2934         free(string);
2935         free(data);
2936         free(instr);
2937         return err;
2938 }
2939
2940 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
2941
2942 static instr_exec_t instruction_table[] = {
2943         [INSTR_RX] = instr_rx_exec,
2944         [INSTR_TX] = instr_tx_exec,
2945
2946         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
2947         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
2948         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
2949         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
2950         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
2951         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
2952         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
2953         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
2954
2955         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
2956         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
2957         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
2958         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
2959         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
2960         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
2961         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
2962         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
2963         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
2964
2965         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
2966         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
2967
2968         [INSTR_MOV] = instr_mov_exec,
2969         [INSTR_MOV_S] = instr_mov_s_exec,
2970         [INSTR_MOV_I] = instr_mov_i_exec,
2971
2972         [INSTR_DMA_HT] = instr_dma_ht_exec,
2973         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
2974         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
2975         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
2976         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
2977         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
2978         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
2979         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
2980 };
2981
2982 static inline void
2983 instr_exec(struct rte_swx_pipeline *p)
2984 {
2985         struct thread *t = &p->threads[p->thread_id];
2986         struct instruction *ip = t->ip;
2987         instr_exec_t instr = instruction_table[ip->type];
2988
2989         instr(p);
2990 }
2991
2992 /*
2993  * Action.
2994  */
2995 static struct action *
2996 action_find(struct rte_swx_pipeline *p, const char *name)
2997 {
2998         struct action *elem;
2999
3000         if (!name)
3001                 return NULL;
3002
3003         TAILQ_FOREACH(elem, &p->actions, node)
3004                 if (strcmp(elem->name, name) == 0)
3005                         return elem;
3006
3007         return NULL;
3008 }
3009
3010 static struct field *
3011 action_field_find(struct action *a, const char *name)
3012 {
3013         return a->st ? struct_type_field_find(a->st, name) : NULL;
3014 }
3015
3016 static struct field *
3017 action_field_parse(struct action *action, const char *name)
3018 {
3019         if (name[0] != 't' || name[1] != '.')
3020                 return NULL;
3021
3022         return action_field_find(action, &name[2]);
3023 }
3024
3025 int
3026 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
3027                                const char *name,
3028                                const char *args_struct_type_name,
3029                                const char **instructions,
3030                                uint32_t n_instructions)
3031 {
3032         struct struct_type *args_struct_type;
3033         struct action *a;
3034         int err;
3035
3036         CHECK(p, EINVAL);
3037
3038         CHECK_NAME(name, EINVAL);
3039         CHECK(!action_find(p, name), EEXIST);
3040
3041         if (args_struct_type_name) {
3042                 CHECK_NAME(args_struct_type_name, EINVAL);
3043                 args_struct_type = struct_type_find(p, args_struct_type_name);
3044                 CHECK(args_struct_type, EINVAL);
3045         } else {
3046                 args_struct_type = NULL;
3047         }
3048
3049         /* Node allocation. */
3050         a = calloc(1, sizeof(struct action));
3051         CHECK(a, ENOMEM);
3052
3053         /* Node initialization. */
3054         strcpy(a->name, name);
3055         a->st = args_struct_type;
3056         a->id = p->n_actions;
3057
3058         /* Instruction translation. */
3059         err = instruction_config(p, a, instructions, n_instructions);
3060         if (err) {
3061                 free(a);
3062                 return err;
3063         }
3064
3065         /* Node add to tailq. */
3066         TAILQ_INSERT_TAIL(&p->actions, a, node);
3067         p->n_actions++;
3068
3069         return 0;
3070 }
3071
3072 static int
3073 action_build(struct rte_swx_pipeline *p)
3074 {
3075         struct action *action;
3076
3077         p->action_instructions = calloc(p->n_actions,
3078                                         sizeof(struct instruction *));
3079         CHECK(p->action_instructions, ENOMEM);
3080
3081         TAILQ_FOREACH(action, &p->actions, node)
3082                 p->action_instructions[action->id] = action->instructions;
3083
3084         return 0;
3085 }
3086
3087 static void
3088 action_build_free(struct rte_swx_pipeline *p)
3089 {
3090         free(p->action_instructions);
3091         p->action_instructions = NULL;
3092 }
3093
3094 static void
3095 action_free(struct rte_swx_pipeline *p)
3096 {
3097         action_build_free(p);
3098
3099         for ( ; ; ) {
3100                 struct action *action;
3101
3102                 action = TAILQ_FIRST(&p->actions);
3103                 if (!action)
3104                         break;
3105
3106                 TAILQ_REMOVE(&p->actions, action, node);
3107                 free(action->instructions);
3108                 free(action);
3109         }
3110 }
3111
3112 /*
3113  * Table.
3114  */
3115 static struct table_type *
3116 table_type_find(struct rte_swx_pipeline *p, const char *name)
3117 {
3118         struct table_type *elem;
3119
3120         TAILQ_FOREACH(elem, &p->table_types, node)
3121                 if (strcmp(elem->name, name) == 0)
3122                         return elem;
3123
3124         return NULL;
3125 }
3126
3127 static struct table_type *
3128 table_type_resolve(struct rte_swx_pipeline *p,
3129                    const char *recommended_type_name,
3130                    enum rte_swx_table_match_type match_type)
3131 {
3132         struct table_type *elem;
3133
3134         /* Only consider the recommended type if the match type is correct. */
3135         if (recommended_type_name)
3136                 TAILQ_FOREACH(elem, &p->table_types, node)
3137                         if (!strcmp(elem->name, recommended_type_name) &&
3138                             (elem->match_type == match_type))
3139                                 return elem;
3140
3141         /* Ignore the recommended type and get the first element with this match
3142          * type.
3143          */
3144         TAILQ_FOREACH(elem, &p->table_types, node)
3145                 if (elem->match_type == match_type)
3146                         return elem;
3147
3148         return NULL;
3149 }
3150
3151 static struct table *
3152 table_find(struct rte_swx_pipeline *p, const char *name)
3153 {
3154         struct table *elem;
3155
3156         TAILQ_FOREACH(elem, &p->tables, node)
3157                 if (strcmp(elem->name, name) == 0)
3158                         return elem;
3159
3160         return NULL;
3161 }
3162
3163 static struct table *
3164 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
3165 {
3166         struct table *table = NULL;
3167
3168         TAILQ_FOREACH(table, &p->tables, node)
3169                 if (table->id == id)
3170                         return table;
3171
3172         return NULL;
3173 }
3174
3175 int
3176 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
3177                                      const char *name,
3178                                      enum rte_swx_table_match_type match_type,
3179                                      struct rte_swx_table_ops *ops)
3180 {
3181         struct table_type *elem;
3182
3183         CHECK(p, EINVAL);
3184
3185         CHECK_NAME(name, EINVAL);
3186         CHECK(!table_type_find(p, name), EEXIST);
3187
3188         CHECK(ops, EINVAL);
3189         CHECK(ops->create, EINVAL);
3190         CHECK(ops->lkp, EINVAL);
3191         CHECK(ops->free, EINVAL);
3192
3193         /* Node allocation. */
3194         elem = calloc(1, sizeof(struct table_type));
3195         CHECK(elem, ENOMEM);
3196
3197         /* Node initialization. */
3198         strcpy(elem->name, name);
3199         elem->match_type = match_type;
3200         memcpy(&elem->ops, ops, sizeof(*ops));
3201
3202         /* Node add to tailq. */
3203         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
3204
3205         return 0;
3206 }
3207
3208 static enum rte_swx_table_match_type
3209 table_match_type_resolve(struct rte_swx_match_field_params *fields,
3210                          uint32_t n_fields)
3211 {
3212         uint32_t i;
3213
3214         for (i = 0; i < n_fields; i++)
3215                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
3216                         break;
3217
3218         if (i == n_fields)
3219                 return RTE_SWX_TABLE_MATCH_EXACT;
3220
3221         if ((i == n_fields - 1) &&
3222             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
3223                 return RTE_SWX_TABLE_MATCH_LPM;
3224
3225         return RTE_SWX_TABLE_MATCH_WILDCARD;
3226 }
3227
3228 int
3229 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
3230                               const char *name,
3231                               struct rte_swx_pipeline_table_params *params,
3232                               const char *recommended_table_type_name,
3233                               const char *args,
3234                               uint32_t size)
3235 {
3236         struct table_type *type;
3237         struct table *t;
3238         struct action *default_action;
3239         struct header *header = NULL;
3240         int is_header = 0;
3241         uint32_t offset_prev = 0, action_data_size_max = 0, i;
3242
3243         CHECK(p, EINVAL);
3244
3245         CHECK_NAME(name, EINVAL);
3246         CHECK(!table_find(p, name), EEXIST);
3247
3248         CHECK(params, EINVAL);
3249
3250         /* Match checks. */
3251         CHECK(!params->n_fields || params->fields, EINVAL);
3252         for (i = 0; i < params->n_fields; i++) {
3253                 struct rte_swx_match_field_params *field = &params->fields[i];
3254                 struct header *h;
3255                 struct field *hf, *mf;
3256                 uint32_t offset;
3257
3258                 CHECK_NAME(field->name, EINVAL);
3259
3260                 hf = header_field_parse(p, field->name, &h);
3261                 mf = metadata_field_parse(p, field->name);
3262                 CHECK(hf || mf, EINVAL);
3263
3264                 offset = hf ? hf->offset : mf->offset;
3265
3266                 if (i == 0) {
3267                         is_header = hf ? 1 : 0;
3268                         header = hf ? h : NULL;
3269                         offset_prev = offset;
3270
3271                         continue;
3272                 }
3273
3274                 CHECK((is_header && hf && (h->id == header->id)) ||
3275                       (!is_header && mf), EINVAL);
3276
3277                 CHECK(offset > offset_prev, EINVAL);
3278                 offset_prev = offset;
3279         }
3280
3281         /* Action checks. */
3282         CHECK(params->n_actions, EINVAL);
3283         CHECK(params->action_names, EINVAL);
3284         for (i = 0; i < params->n_actions; i++) {
3285                 const char *action_name = params->action_names[i];
3286                 struct action *a;
3287                 uint32_t action_data_size;
3288
3289                 CHECK(action_name, EINVAL);
3290
3291                 a = action_find(p, action_name);
3292                 CHECK(a, EINVAL);
3293
3294                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
3295                 if (action_data_size > action_data_size_max)
3296                         action_data_size_max = action_data_size;
3297         }
3298
3299         CHECK(params->default_action_name, EINVAL);
3300         for (i = 0; i < p->n_actions; i++)
3301                 if (!strcmp(params->action_names[i],
3302                             params->default_action_name))
3303                         break;
3304         CHECK(i < params->n_actions, EINVAL);
3305         default_action = action_find(p, params->default_action_name);
3306         CHECK((default_action->st && params->default_action_data) ||
3307               !params->default_action_data, EINVAL);
3308
3309         /* Table type checks. */
3310         if (params->n_fields) {
3311                 enum rte_swx_table_match_type match_type;
3312
3313                 match_type = table_match_type_resolve(params->fields,
3314                                                       params->n_fields);
3315                 type = table_type_resolve(p,
3316                                           recommended_table_type_name,
3317                                           match_type);
3318                 CHECK(type, EINVAL);
3319         } else {
3320                 type = NULL;
3321         }
3322
3323         /* Memory allocation. */
3324         t = calloc(1, sizeof(struct table));
3325         CHECK(t, ENOMEM);
3326
3327         t->fields = calloc(params->n_fields, sizeof(struct match_field));
3328         if (!t->fields) {
3329                 free(t);
3330                 CHECK(0, ENOMEM);
3331         }
3332
3333         t->actions = calloc(params->n_actions, sizeof(struct action *));
3334         if (!t->actions) {
3335                 free(t->fields);
3336                 free(t);
3337                 CHECK(0, ENOMEM);
3338         }
3339
3340         if (action_data_size_max) {
3341                 t->default_action_data = calloc(1, action_data_size_max);
3342                 if (!t->default_action_data) {
3343                         free(t->actions);
3344                         free(t->fields);
3345                         free(t);
3346                         CHECK(0, ENOMEM);
3347                 }
3348         }
3349
3350         /* Node initialization. */
3351         strcpy(t->name, name);
3352         if (args && args[0])
3353                 strcpy(t->args, args);
3354         t->type = type;
3355
3356         for (i = 0; i < params->n_fields; i++) {
3357                 struct rte_swx_match_field_params *field = &params->fields[i];
3358                 struct match_field *f = &t->fields[i];
3359
3360                 f->match_type = field->match_type;
3361                 f->field = is_header ?
3362                         header_field_parse(p, field->name, NULL) :
3363                         metadata_field_parse(p, field->name);
3364         }
3365         t->n_fields = params->n_fields;
3366         t->is_header = is_header;
3367         t->header = header;
3368
3369         for (i = 0; i < params->n_actions; i++)
3370                 t->actions[i] = action_find(p, params->action_names[i]);
3371         t->default_action = default_action;
3372         if (default_action->st)
3373                 memcpy(t->default_action_data,
3374                        params->default_action_data,
3375                        default_action->st->n_bits / 8);
3376         t->n_actions = params->n_actions;
3377         t->default_action_is_const = params->default_action_is_const;
3378         t->action_data_size_max = action_data_size_max;
3379
3380         t->size = size;
3381         t->id = p->n_tables;
3382
3383         /* Node add to tailq. */
3384         TAILQ_INSERT_TAIL(&p->tables, t, node);
3385         p->n_tables++;
3386
3387         return 0;
3388 }
3389
3390 static struct rte_swx_table_params *
3391 table_params_get(struct table *table)
3392 {
3393         struct rte_swx_table_params *params;
3394         struct field *first, *last;
3395         uint8_t *key_mask;
3396         uint32_t key_size, key_offset, action_data_size, i;
3397
3398         /* Memory allocation. */
3399         params = calloc(1, sizeof(struct rte_swx_table_params));
3400         if (!params)
3401                 return NULL;
3402
3403         /* Key offset and size. */
3404         first = table->fields[0].field;
3405         last = table->fields[table->n_fields - 1].field;
3406         key_offset = first->offset / 8;
3407         key_size = (last->offset + last->n_bits - first->offset) / 8;
3408
3409         /* Memory allocation. */
3410         key_mask = calloc(1, key_size);
3411         if (!key_mask) {
3412                 free(params);
3413                 return NULL;
3414         }
3415
3416         /* Key mask. */
3417         for (i = 0; i < table->n_fields; i++) {
3418                 struct field *f = table->fields[i].field;
3419                 uint32_t start = (f->offset - first->offset) / 8;
3420                 size_t size = f->n_bits / 8;
3421
3422                 memset(&key_mask[start], 0xFF, size);
3423         }
3424
3425         /* Action data size. */
3426         action_data_size = 0;
3427         for (i = 0; i < table->n_actions; i++) {
3428                 struct action *action = table->actions[i];
3429                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
3430
3431                 if (ads > action_data_size)
3432                         action_data_size = ads;
3433         }
3434
3435         /* Fill in. */
3436         params->match_type = table->type->match_type;
3437         params->key_size = key_size;
3438         params->key_offset = key_offset;
3439         params->key_mask0 = key_mask;
3440         params->action_data_size = action_data_size;
3441         params->n_keys_max = table->size;
3442
3443         return params;
3444 }
3445
3446 static void
3447 table_params_free(struct rte_swx_table_params *params)
3448 {
3449         if (!params)
3450                 return;
3451
3452         free(params->key_mask0);
3453         free(params);
3454 }
3455
3456 static int
3457 table_state_build(struct rte_swx_pipeline *p)
3458 {
3459         struct table *table;
3460
3461         p->table_state = calloc(p->n_tables,
3462                                 sizeof(struct rte_swx_table_state));
3463         CHECK(p->table_state, ENOMEM);
3464
3465         TAILQ_FOREACH(table, &p->tables, node) {
3466                 struct rte_swx_table_state *ts = &p->table_state[table->id];
3467
3468                 if (table->type) {
3469                         struct rte_swx_table_params *params;
3470
3471                         /* ts->obj. */
3472                         params = table_params_get(table);
3473                         CHECK(params, ENOMEM);
3474
3475                         ts->obj = table->type->ops.create(params,
3476                                 NULL,
3477                                 table->args,
3478                                 p->numa_node);
3479
3480                         table_params_free(params);
3481                         CHECK(ts->obj, ENODEV);
3482                 }
3483
3484                 /* ts->default_action_data. */
3485                 if (table->action_data_size_max) {
3486                         ts->default_action_data =
3487                                 malloc(table->action_data_size_max);
3488                         CHECK(ts->default_action_data, ENOMEM);
3489
3490                         memcpy(ts->default_action_data,
3491                                table->default_action_data,
3492                                table->action_data_size_max);
3493                 }
3494
3495                 /* ts->default_action_id. */
3496                 ts->default_action_id = table->default_action->id;
3497         }
3498
3499         return 0;
3500 }
3501
3502 static void
3503 table_state_build_free(struct rte_swx_pipeline *p)
3504 {
3505         uint32_t i;
3506
3507         if (!p->table_state)
3508                 return;
3509
3510         for (i = 0; i < p->n_tables; i++) {
3511                 struct rte_swx_table_state *ts = &p->table_state[i];
3512                 struct table *table = table_find_by_id(p, i);
3513
3514                 /* ts->obj. */
3515                 if (table->type && ts->obj)
3516                         table->type->ops.free(ts->obj);
3517
3518                 /* ts->default_action_data. */
3519                 free(ts->default_action_data);
3520         }
3521
3522         free(p->table_state);
3523         p->table_state = NULL;
3524 }
3525
3526 static void
3527 table_state_free(struct rte_swx_pipeline *p)
3528 {
3529         table_state_build_free(p);
3530 }
3531
3532 static int
3533 table_stub_lkp(void *table __rte_unused,
3534                void *mailbox __rte_unused,
3535                uint8_t **key __rte_unused,
3536                uint64_t *action_id __rte_unused,
3537                uint8_t **action_data __rte_unused,
3538                int *hit)
3539 {
3540         *hit = 0;
3541         return 1; /* DONE. */
3542 }
3543
3544 static int
3545 table_build(struct rte_swx_pipeline *p)
3546 {
3547         uint32_t i;
3548
3549         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
3550                 struct thread *t = &p->threads[i];
3551                 struct table *table;
3552
3553                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
3554                 CHECK(t->tables, ENOMEM);
3555
3556                 TAILQ_FOREACH(table, &p->tables, node) {
3557                         struct table_runtime *r = &t->tables[table->id];
3558
3559                         if (table->type) {
3560                                 uint64_t size;
3561
3562                                 size = table->type->ops.mailbox_size_get();
3563
3564                                 /* r->func. */
3565                                 r->func = table->type->ops.lkp;
3566
3567                                 /* r->mailbox. */
3568                                 if (size) {
3569                                         r->mailbox = calloc(1, size);
3570                                         CHECK(r->mailbox, ENOMEM);
3571                                 }
3572
3573                                 /* r->key. */
3574                                 r->key = table->is_header ?
3575                                         &t->structs[table->header->struct_id] :
3576                                         &t->structs[p->metadata_struct_id];
3577                         } else {
3578                                 r->func = table_stub_lkp;
3579                         }
3580                 }
3581         }
3582
3583         return 0;
3584 }
3585
3586 static void
3587 table_build_free(struct rte_swx_pipeline *p)
3588 {
3589         uint32_t i;
3590
3591         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
3592                 struct thread *t = &p->threads[i];
3593                 uint32_t j;
3594
3595                 if (!t->tables)
3596                         continue;
3597
3598                 for (j = 0; j < p->n_tables; j++) {
3599                         struct table_runtime *r = &t->tables[j];
3600
3601                         free(r->mailbox);
3602                 }
3603
3604                 free(t->tables);
3605                 t->tables = NULL;
3606         }
3607 }
3608
3609 static void
3610 table_free(struct rte_swx_pipeline *p)
3611 {
3612         table_build_free(p);
3613
3614         /* Tables. */
3615         for ( ; ; ) {
3616                 struct table *elem;
3617
3618                 elem = TAILQ_FIRST(&p->tables);
3619                 if (!elem)
3620                         break;
3621
3622                 TAILQ_REMOVE(&p->tables, elem, node);
3623                 free(elem->fields);
3624                 free(elem->actions);
3625                 free(elem->default_action_data);
3626                 free(elem);
3627         }
3628
3629         /* Table types. */
3630         for ( ; ; ) {
3631                 struct table_type *elem;
3632
3633                 elem = TAILQ_FIRST(&p->table_types);
3634                 if (!elem)
3635                         break;
3636
3637                 TAILQ_REMOVE(&p->table_types, elem, node);
3638                 free(elem);
3639         }
3640 }
3641
3642 /*
3643  * Pipeline.
3644  */
3645 int
3646 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
3647 {
3648         struct rte_swx_pipeline *pipeline;
3649
3650         /* Check input parameters. */
3651         CHECK(p, EINVAL);
3652
3653         /* Memory allocation. */
3654         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
3655         CHECK(pipeline, ENOMEM);
3656
3657         /* Initialization. */
3658         TAILQ_INIT(&pipeline->struct_types);
3659         TAILQ_INIT(&pipeline->port_in_types);
3660         TAILQ_INIT(&pipeline->ports_in);
3661         TAILQ_INIT(&pipeline->port_out_types);
3662         TAILQ_INIT(&pipeline->ports_out);
3663         TAILQ_INIT(&pipeline->extern_types);
3664         TAILQ_INIT(&pipeline->extern_objs);
3665         TAILQ_INIT(&pipeline->extern_funcs);
3666         TAILQ_INIT(&pipeline->headers);
3667         TAILQ_INIT(&pipeline->actions);
3668         TAILQ_INIT(&pipeline->table_types);
3669         TAILQ_INIT(&pipeline->tables);
3670
3671         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
3672         pipeline->numa_node = numa_node;
3673
3674         *p = pipeline;
3675         return 0;
3676 }
3677
3678 void
3679 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
3680 {
3681         if (!p)
3682                 return;
3683
3684         free(p->instructions);
3685
3686         table_state_free(p);
3687         table_free(p);
3688         action_free(p);
3689         metadata_free(p);
3690         header_free(p);
3691         extern_func_free(p);
3692         extern_obj_free(p);
3693         port_out_free(p);
3694         port_in_free(p);
3695         struct_free(p);
3696
3697         free(p);
3698 }
3699
3700 int
3701 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
3702                                      const char **instructions,
3703                                      uint32_t n_instructions)
3704 {
3705         int err;
3706         uint32_t i;
3707
3708         err = instruction_config(p, NULL, instructions, n_instructions);
3709         if (err)
3710                 return err;
3711
3712         /* Thread instruction pointer reset. */
3713         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
3714                 struct thread *t = &p->threads[i];
3715
3716                 thread_ip_reset(p, t);
3717         }
3718
3719         return 0;
3720 }
3721
3722 int
3723 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
3724 {
3725         int status;
3726
3727         CHECK(p, EINVAL);
3728         CHECK(p->build_done == 0, EEXIST);
3729
3730         status = port_in_build(p);
3731         if (status)
3732                 goto error;
3733
3734         status = port_out_build(p);
3735         if (status)
3736                 goto error;
3737
3738         status = struct_build(p);
3739         if (status)
3740                 goto error;
3741
3742         status = extern_obj_build(p);
3743         if (status)
3744                 goto error;
3745
3746         status = extern_func_build(p);
3747         if (status)
3748                 goto error;
3749
3750         status = header_build(p);
3751         if (status)
3752                 goto error;
3753
3754         status = metadata_build(p);
3755         if (status)
3756                 goto error;
3757
3758         status = action_build(p);
3759         if (status)
3760                 goto error;
3761
3762         status = table_build(p);
3763         if (status)
3764                 goto error;
3765
3766         status = table_state_build(p);
3767         if (status)
3768                 goto error;
3769
3770         p->build_done = 1;
3771         return 0;
3772
3773 error:
3774         table_state_build_free(p);
3775         table_build_free(p);
3776         action_build_free(p);
3777         metadata_build_free(p);
3778         header_build_free(p);
3779         extern_func_build_free(p);
3780         extern_obj_build_free(p);
3781         port_out_build_free(p);
3782         port_in_build_free(p);
3783         struct_build_free(p);
3784
3785         return status;
3786 }
3787
3788 void
3789 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
3790 {
3791         uint32_t i;
3792
3793         for (i = 0; i < n_instructions; i++)
3794                 instr_exec(p);
3795 }
3796
3797 /*
3798  * Control.
3799  */
3800 int
3801 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
3802                                  struct rte_swx_table_state **table_state)
3803 {
3804         if (!p || !table_state || !p->build_done)
3805                 return -EINVAL;
3806
3807         *table_state = p->table_state;
3808         return 0;
3809 }
3810
3811 int
3812 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
3813                                  struct rte_swx_table_state *table_state)
3814 {
3815         if (!p || !table_state || !p->build_done)
3816                 return -EINVAL;
3817
3818         p->table_state = table_state;
3819         return 0;
3820 }